diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 35d7022..ef746d9 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -14,4 +14,5 @@ jobs: VALIDATE_JSON: false VALIDATE_HTML: false VALIDATE_CSS: false - VALIDATE_BASH_EXEC: false \ No newline at end of file + VALIDATE_JAVASCRIPT_ES: false + VALIDATE_BASH_EXEC: false diff --git a/.github/workflows/post-pr-reviews.yml b/.github/workflows/post-pr-reviews.yml index b026cf3..2242a95 100644 --- a/.github/workflows/post-pr-reviews.yml +++ b/.github/workflows/post-pr-reviews.yml @@ -1,3 +1,4 @@ +--- name: Post PR code suggestions on: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb5bf04..71e660b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,8 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Replace run integration test command run: | - sed -i "s+mundialis/actinia:latest+mundialis/actinia:grass8.3+g" docker/actinia-module-plugin-test/Dockerfile + sed -i "s+mundialis/actinia:latest+mundialis/actinia:grass8.3+g" \ + docker/actinia-module-plugin-test/Dockerfile - name: Tests of actinia-module-plugin id: docker_build uses: docker/build-push-action@v6 diff --git a/actinia-module.md b/actinia-module.md index ef47a3c..98d5893 100644 --- a/actinia-module.md +++ b/actinia-module.md @@ -204,12 +204,24 @@ One example response to describe the GRASS GIS module "v.buffer" looks like this ``` -## 2. Process-chain templates +## 2. Actinia modules (Process-chain templates) An example process-chain looks like this: ```json { "list": [ + { + "id": "copy_vector_data", + "module": "g.copy", + "comment": "Copies a map", + "inputs": [ + { + "comment": "name of map to be copied and name of new copy.", + "param": "vector", + "value": "polygons_old,polygons" + } + ] + }, { "id": "add_column_for_area", "module": "v.db.addcolumn", @@ -239,10 +251,6 @@ An example process-chain looks like this: { "param": "option", "value": "area" - }, - { - "param": "unit", - "value": "h" } ] }, @@ -278,13 +286,25 @@ Therefore actinia-module-plugin is able to store process-chain templates and the "description": "Computes the areas in h for selected map.", "template": { "list": [ + { + "id": "copy_vector_data", + "module": "g.copy", + "comment": "Copies a map", + "inputs": [ + { + "comment": "name of map to be copied and name of new copy.", + "param": "vector", + "value": "{{ name_and_copy }}" + } + ] + }, { "id": "add_column_for_area", "module": "v.db.addcolumn", "inputs": [ { "param": "map", - "value": "{{ name }}" + "value": "{{ copy }}" }, { "param": "columns", @@ -298,7 +318,7 @@ Therefore actinia-module-plugin is able to store process-chain templates and the "inputs": [ { "param": "map", - "value": "{{ name }}" + "value": "{{ copy }}" }, { "param": "columns", @@ -306,11 +326,7 @@ Therefore actinia-module-plugin is able to store process-chain templates and the }, { "param": "option", - "value": "area" - }, - { - "param": "unit", - "value": "h" + "value": "{{ option }}" } ] }, @@ -320,7 +336,7 @@ Therefore actinia-module-plugin is able to store process-chain templates and the "inputs": [ { "param": "map", - "value": "{{ name }}" + "value": "{{ copy }}" }, { "param": "columns", @@ -335,7 +351,10 @@ Therefore actinia-module-plugin is able to store process-chain templates and the ``` -This way, the user only needs to set values for the defined variables. The user doesn't even need to know the whole template. Actinia-module-plugin will translate it into one actinia-module, exploiting only the values necessary for input: +This way, the user only needs to set values for the defined variables. The user doesn't even need to know the whole template. Actinia-module-plugin will translate it into one actinia-module, exploiting only the values necessary for input. +For the user this looks like one module in the process-chain and can use this process-chain to start a process. +The endpoints to use templating in processing are a bit different to the normal actinia-core processing endpoints: +`/projects//processing_export` for ephemeral processing and `/projects//mapsets//processing` for persisten processing: ```json { @@ -345,107 +364,149 @@ This way, the user only needs to set values for the defined variables. The user "module": "vector_area", "inputs": [ { - "param": "v.db.addcolumn_map", - "value": "{{ name }}" - }, - { - "param": "v.db.addcolumn_columns", - "value": "{{ columns }}" + "param": "name_and_copy", + "value": "polygons_old,polygons" }, { - "param": "v.to.db_map", - "value": "{{ name }}" + "param": "copy", + "value": "polygons" }, { - "param": "v.to.db_columns", - "value": "{{ column_name }}" + "param": "column", + "value": "area_h DOUBLE PRECISION" }, { - "param": "v.db.select_map", - "value": "{{ name }}" + "param": "column_name", + "value": "area_h" }, { - "param": "v.db.select_columns", - "value": "{{ column_name }}" + "param": "option", + "value": "area" } ] } ], "version": "1" } -``` -For the user this looks like one module in the process-chain. In further development, it will be shrinked even more to not exploit the same variable twice. This might then look like this: +``` -```json -{ - "list": [ - { - "id": "vector_area", - "module": "vector_area", - "inputs": [ - { - "param": "v.db.addcolumn_map", - "value": "{{ name }}" - }, - { - "param": "v.db.addcolumn_columns", - "value": "{{ columns }}" - }, - { - "param": "v.to.db_columns", - "value": "{{ column_name }}" - } - ] - } - ], - "version": "1" - } +## 3. Upload own Templates and Template self-description +There are global and user actinia-modules. Globa actinia-modules need to be available as JSON file inside actinia in a confirgurable directory. +User templates can be created, read, updated and deleted via HTTP REST endpoints: +```bash +CREATE: HTTP POST my_template.json `/actinia_templates` +READ: HTTP GET `/actinia_templates/my_template` +UPDATE: HTTP PUT `/actinia_templates/my_template` +DELETE: HTTP DELETE `/actinia_templates/my_template` ``` -## 3. Template self-description - -Mapping the process-chain template to a self-description will look as follows and the user won't feel the difference between a grass-module and an actinia-module anymore: +Mapping the process-chain template to a self-description will look as follows and the user won't feel the difference between a grass-module and an actinia-module anymore. So requesting `/actinia_modules/vector_area` will return: ```json { "categories": [ - "actinia-module" + "actinia-module", + "global-template" ], - "description": "Copies zipcodes_wake and compute the areas in h.", + "description": "Copies vector and computes the areas in h.", "id": "vector_area", - "parameters": { - "v.db.addcolumn_columns": { - "description": "Name and type of the new column(s) ('name type [,name type, ...]'). Types depend on database backend, but all support VARCHAR(), INT, DOUBLE PRECISION and DATE. Example: 'label varchar(250), value integer'", - "required": true, + "parameters": [ + { + "description": "vector map(s) to be copied. [generated from g.copy_vector] - name of map to be copied and name of new copy.", + "name": "name_and_copy", + "optional": true, "schema": { - "type": "array" + "subtype": "vector", + "type": "string" } }, - "v.db.addcolumn_map": { - "description": "Name of vector map. Or data source for direct OGR access", - "required": true, + { + "description": "Name of vector map. Or data source for direct OGR access. [generated from v.db.addcolumn_map]", + "name": "copy", + "optional": false, "schema": { "subtype": "vector", "type": "string" } }, - "v.to.db_columns": { - "description": "Name of attribute column(s) to populate. Name of attribute column(s)", - "required": true, + { + "description": "Name and type of the new column(s) ('name type [,name type, ...]'). Types depend on database backend, but all support VARCHAR(), INT, DOUBLE PRECISION and DATE. Example: 'label varchar(250), value integer'. [generated from v.db.addcolumn_columns]", + "name": "column", + "optional": false, + "schema": { + "type": "array" + } + }, + { + "description": "Value to upload. [generated from v.to.db_option]", + "name": "option", + "optional": false, + "schema": { + "enum": [ + "cat", + "area", + "compact", + "fd", + "perimeter", + "length", + "count", + "coor", + "start", + "end", + "sides", + "query", + "slope", + "sinuous", + "azimuth", + "bbox" + ], + "type": "string" + } + }, + { + "description": "Name of attribute column(s) to populate. Name of attribute column(s). [generated from v.to.db_columns]", + "name": "column_name", + "optional": false, "schema": { "subtype": "dbcolumn", "type": "array" } } - } + ], + "projects": [], + "returns": [] } ``` +## 4. Processing via actinia_module + +Besides the existing processing endpoints, it is also possible to run ephemeral processing directly at a certain module +`/actinia_modules//process`, e.g. `/actinia_modules/point_in_polygon/process`. + +There are some current limitations: +- Only ephemeral processing is allowed for now +- To tell actinia in which project to run the process, it needs to defined in the template, e.g. +```json +{ + "id": "ndvi", + "description": "Generate NDVI map", + "projects": ["nc_spm_08"], + "template": { +... +``` + in the future, multiple projects can be added comma separated to indicate for which project the process chain template can be used. For now the project to run the process in will be read from the template, so only one project should be defined. +- the schema for the HTTP POST body is not yet defined. It might be close to OGC API Processes. Right now, only a single value can be passed in the HTTP POST body without parameter name. It will then be checked for variable names inside the template and if only one is given, it will be used. If multiple variables are needed, this is not yet implemented as no schema is defined. + Because of this limitation, it is only possible to run processes with templates with exactly ONE variable. + +When the HTTP POST is send, +- the template is loaded, the one value from the request body is filled and the process chain is passed to actinia-core for processing. +- Response is the usual actinia-core response for processes containing resource_id which can be polled. + -## 4. Hints for template creation +## 5. Hints for template creation * __It is not allowed to manipulate existing maps__ This leads to an error for ephemeral and persistent processing. E.g. `v.db.addcolumn` is not working and leads to `"ERROR: Vector map not found in current mapset"`. Therefore copy it first (in the same processchain!) with g.copy. If it is done in a separate process chain and even exists in the user mapset, it is still not working. @@ -454,7 +515,7 @@ This leads to an error for ephemeral and persistent processing. E.g. `v.db.addco On persistent processing, name of map will be first searched in PERSISTENT mapset, then in user mapset. If both exist and user mapset should be used, this can be overwritten by mymap@mymapset. -## 5. Conventions for template creation +## 6. Conventions for template creation * __Only use placeholder for a whole value of a GRASS GIS attribute.__ @@ -477,7 +538,7 @@ TODO discuss TODO discuss -## 6. Overview of endpoints for module self-description and execution +## 7. Overview of endpoints for module self-description and execution List / Describe only GRASS Modules @@ -508,7 +569,7 @@ Full API docs -## 7. Additional Notes +## 8. Additional Notes ### The self-description tries to comply the [openEO API](https://api.openeo.org/#tag/Process-Discovery) where applicable At some points, however, we have to divert from their API: diff --git a/config/templates/pc_templates/point_in_polygon.json b/config/templates/pc_templates/point_in_polygon.json index 12207df..55d2ccd 100644 --- a/config/templates/pc_templates/point_in_polygon.json +++ b/config/templates/pc_templates/point_in_polygon.json @@ -1,6 +1,7 @@ { "id": "point_in_polygon", "description": "Imports point and polygon and checks if point is in polygon.", + "projects": ["nc_spm_08"], "template": { "list": [ { diff --git a/docker/actinia-module-plugin-test/Dockerfile b/docker/actinia-module-plugin-test/Dockerfile index d10ea4b..d3e7251 100644 --- a/docker/actinia-module-plugin-test/Dockerfile +++ b/docker/actinia-module-plugin-test/Dockerfile @@ -13,7 +13,7 @@ ENV TEMPLATE_VALUE_ENV_TYPE=raster # install things only for tests RUN apk add --no-cache valkey valkey-cli -RUN pip install --upgrade setuptools && pip install pytest pytest-cov pwgen +RUN pip install --no-cache-dir --upgrade setuptools && pip install pytest pytest-cov pwgen ENTRYPOINT ["/bin/sh"] CMD ["/src/start.sh"] diff --git a/docker/release/Dockerfile b/docker/release/Dockerfile deleted file mode 100644 index 7d08e63..0000000 --- a/docker/release/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM alpine:edge AS build - -RUN apk add git python3 postgresql-dev gcc python3-dev musl-dev - -RUN pip3 install --upgrade pip pep517 wheel - -# Build -COPY . /src/actinia-module-plugin -WORKDIR /src/actinia-module-plugin -# TODO: include tests. Currently only running whith actinia_core installed -# RUN pip3 install -r requirements.txt && python3 setup.py test -# build including dependency list from setup.cfg (automatically installed). -# pip3 wheel creates wheels out of all requirements for offline usage -RUN python3 -m pep517.build --out-dir /build . && \ - pip3 wheel -r requirements.txt -w /build - - -FROM alpine:edge - -COPY --from=build /build/actinia*.whl /build/ -COPY docker/release/create_release_with_asset.sh . - -ARG release_url=https://api.github.com/repos/mundialis/actinia-module-plugin/releases -ARG tag=0.0 -ARG credentials=dummy:dummy -ARG file=/build/actinia*.whl - -ENV env_release_url=$release_url -ENV env_tag=$tag -ENV env_credentials=$credentials -ENV env_file=$file - -RUN apk add --no-cache curl jq - -ENTRYPOINT ["/bin/sh"] -CMD ["./create_release_with_asset.sh"] diff --git a/docker/release/README.md b/docker/release/README.md deleted file mode 100644 index bef0b68..0000000 --- a/docker/release/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Only kept for reference - currently travis takes care of building the wheel - -To create a new release, you can use this docker to build the wheel and add -to the release. - -The `docker build` command builds the wheel and integrated the build-args for -the github release. - -The `docker run` command creates the build and uploads the -previously created wheel as asset. - - -```bash -git clone git@github.com:mundialis/actinia-module-plugin.git -cd actinia-module-plugin - -tag=0.0 -credentials=mygithubuser:mygithubpw - -docker build --file docker/release/Dockerfile --build-arg tag=$tag --build-arg credentials=$credentials --tag actinia-module-plugin:build . - -docker run --rm actinia-module-plugin:build -``` diff --git a/docker/release/create_release_with_asset.sh b/docker/release/create_release_with_asset.sh deleted file mode 100644 index 437622f..0000000 --- a/docker/release/create_release_with_asset.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -# needed to be set as ENVs before: -#env_release_url -#env_tag -#env_credentials -#env_file - -filefull=`ls ${env_file:-}` -filename=$(basename $filefull) - - -echo "{\"tag_name\": \"${env_tag:-}\",\"target_commitish\": \"master\",\"name\":\"${env_tag:-}\",\"body\": \"Automatically created by CI\",\"draft\": false,\"prerelease\": false}" > /tmp/release_payload.json - -# Create release -curl -u ${env_credentials:-} -X POST -H 'Content-Type: application/json' -d @/tmp/release_payload.json ${env_release_url:-} > resp.json && cat resp.json - -# parse response to create upload_url -upload_url=`cat resp.json | jq '.upload_url' | tr -d '"' | cut -d '{' -f1` -url="${upload_url}?name=${filename}" - -if [ "${upload_url}" = "null" ] -then - echo "Failed to create release, aborting." - exit 1 -fi - -curl -u ${env_credentials:-} -H "Accept: application/vnd.github.manifold-preview" -H "Content-Type: application/zip" --data-binary @$filefull "$url" > resp.json && cat resp.json diff --git a/src/actinia_module_plugin/api/modules/actinia.py b/src/actinia_module_plugin/api/modules/actinia.py index f9dc078..645711d 100644 --- a/src/actinia_module_plugin/api/modules/actinia.py +++ b/src/actinia_module_plugin/api/modules/actinia.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-2021 mundialis GmbH & Co. KG +Copyright (c) 2018-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,13 +25,14 @@ __license__ = "Apache-2.0" __author__ = "Carmen Tawalika" -__copyright__ = "Copyright 2019, mundialis" +__copyright__ = "Copyright 2019-2025, mundialis" __maintainer__ = "Carmen Tawalika" from flask import jsonify, make_response from flask_restful_swagger_2 import swagger from flask_restful import Resource + from actinia_core.rest.base.resource_base import ResourceBase from actinia_module_plugin.apidocs import modules diff --git a/src/actinia_module_plugin/api/modules/actinia_process.py b/src/actinia_module_plugin/api/modules/actinia_process.py new file mode 100644 index 0000000..1aac583 --- /dev/null +++ b/src/actinia_module_plugin/api/modules/actinia_process.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2025 mundialis GmbH & Co. KG + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +actinia-module viewer +Templates can be stored file based and in kvdb + +* List all actinia-modules +* Describe single actinia-module +""" + +__license__ = "Apache-2.0" +__author__ = "Carmen Tawalika" +__copyright__ = "Copyright 2025, mundialis" +__maintainer__ = "Carmen Tawalika" + + +from flask import jsonify, make_response +import pickle + +from actinia_core.core.common.kvdb_interface import enqueue_job +from actinia_core.processing.common.ephemeral_processing_with_export import ( + start_job as start_job_ephemeral_processing_with_export, +) +from actinia_core.rest.base.resource_base import ResourceBase + +from actinia_module_plugin.core.common import ( + fillTemplateFromProcessChain, +) +from actinia_module_plugin.core.modules.actinia_common import ( + createActiniaModule, +) +from actinia_module_plugin.core.common import ( + get_user_template_source, + get_global_template_source, +) +from actinia_module_plugin.core.template_parameters import ( + get_template_undef, +) + + +def preprocess_load_tpl_and_enqueue( + self, preprocess_kwargs, start_job, actiniamodule +): + """ + This method looks up the stored process chain template. + Template values are filled according to input values. + The process chain is then passed to actinia-core. + """ + + # run preprocess again after createModuleList + rdc = self.preprocess(**preprocess_kwargs) + + if rdc: + rdc.set_storage_model_to_file() + + tpl_source = get_user_template_source( + actiniamodule + ) or get_global_template_source(actiniamodule) + undef = get_template_undef(tpl_source) + + # TODO parse request data when schema is defined + # Might be close to OGC API processes + kwargs = {} + kwargs[next(iter(undef))] = rdc.request_data + + new_pc = fillTemplateFromProcessChain(actiniamodule, kwargs) + rdc.request_data = new_pc + + enqueue_job(self.job_timeout, start_job, rdc) + + +class ProcessActiniaModule(ResourceBase): + """ + Process process chain template as actinia-module. + + Contains HTTP POST endpoint + Contains swagger documentation + """ + + # TODO: Define input + # @swagger.doc(modules.processActiniaModule_post_docs) + def post(self, actiniamodule): + """Process an actinia module (process chain template).""" + + preprocess_kwargs = {} + preprocess_kwargs["has_json"] = True + # TODO: Currently no project can be read out of request body. + # Instead it will take the first project listed in actinia module + # To be sure only write a single project inside template. + virtual_module = createActiniaModule(self, actiniamodule) + preprocess_kwargs["project_name"] = virtual_module["projects"][0] + + start_job = start_job_ephemeral_processing_with_export + + preprocess_load_tpl_and_enqueue( + self, preprocess_kwargs, start_job, actiniamodule + ) + + html_code, response_model = pickle.loads(self.response_data) + return make_response(jsonify(response_model), html_code) diff --git a/src/actinia_module_plugin/api/processing.py b/src/actinia_module_plugin/api/processing.py index e0b4344..a95d46b 100644 --- a/src/actinia_module_plugin/api/processing.py +++ b/src/actinia_module_plugin/api/processing.py @@ -21,7 +21,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2019-present Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2019-2025 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,7 +44,7 @@ __license__ = "Apache-2.0" __author__ = "Anika Bettge, Sören Gebbert" -__copyright__ = "Copyright 2016-2019, Sören Gebbert, mundialis GmbH & Co. KG" +__copyright__ = "Copyright 2016-2025, Sören Gebbert, mundialis GmbH & Co. KG" __maintainer__ = "mundialis" @@ -69,6 +69,7 @@ post_doc as SCHEMA_DOC_PERSISTENT_PROCESSING, ) +from actinia_module_plugin.core.common import fillTemplateFromProcessChain from actinia_module_plugin.core.modules.actinia_global_templates import ( createProcessChainTemplateListFromFileSystem, ) @@ -76,7 +77,9 @@ createProcessChainTemplateListFromKvdb, ) from actinia_module_plugin.core.modules.grass import createModuleList -from actinia_module_plugin.core.processing import fillTemplateFromProcessChain +from actinia_module_plugin.core.processing import ( + build_kwargs_for_template_rendering, +) def log_error_to_resource_logger(self, msg, rdc): @@ -121,7 +124,11 @@ def set_actinia_modules( elif name in grass_module_list: new_pc.append(module) elif name in actinia_module_list: - module_pc = fillTemplateFromProcessChain(module) + kwargs = build_kwargs_for_template_rendering(module) + module_pc = fillTemplateFromProcessChain( + module["module"], + kwargs, + )["list"] if isinstance(module_pc, str): # then return value is a missing attribute msg = ( diff --git a/src/actinia_module_plugin/core/common.py b/src/actinia_module_plugin/core/common.py index abf6dfa..1d0a60e 100644 --- a/src/actinia_module_plugin/core/common.py +++ b/src/actinia_module_plugin/core/common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-2021 mundialis GmbH & Co. KG +Copyright (c) 2018-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,17 +20,30 @@ """ __author__ = "Carmen Tawalika" -__copyright__ = "2018-2021 mundialis GmbH & Co. KG" +__copyright__ = "2018-2025 mundialis GmbH & Co. KG" __license__ = "Apache-2.0" import json from jinja2 import Template, DictLoader, Environment +from os import environ as env from actinia_module_plugin.core.templates.user_templates import readTemplate +from actinia_module_plugin.core.template_parameters import ( + get_not_needed_params, + get_template_undef, +) +from actinia_module_plugin.resources.logging import log from actinia_module_plugin.resources.templating import pcTplEnv +ENV = { + key.replace("TEMPLATE_VALUE_", ""): val + for key, val in env.items() + if key.startswith("TEMPLATE_VALUE_") +} + + def start_job(timeout, func, *args): """ Execute the provided function in a subprocess @@ -98,3 +111,68 @@ def get_global_template_source(name): tplPath = get_global_template_path(name) tpl_source = pcTplEnv.loader.get_source(pcTplEnv, tplPath)[0] return tpl_source + + +def check_for_errors(undef, parsed_content, tpl_source, kwargs): + """ + This method checks if all placeholders are filled with values and + returns the placeholder if missing. Exceptions are default values for which + the given default value can be used and if statements for which the value + can be empty. + """ + # find default variables from processchain and variables which are only in + # an if statement and has not to be set + not_needed_vars = get_not_needed_params(undef, tpl_source, parsed_content) + + for i in undef: + # check if undef variables are needed or set in the kwargs + if i not in kwargs.keys() and i not in not_needed_vars: + log.error('Required parameter "' + i + '" not in process chain!') + return i + + return None + + +def fill_env_values(filled_params, undef): + """ + This function checks if a undefined variable is set in the environment + variables and set it in kwargs if not already set. + """ + if len(ENV) > 0: + for param in undef: + if param not in filled_params and param.upper() in ENV: + filled_params[param] = ENV[param.upper()] + + +def fillTemplateFromProcessChain(actiniamodulename, kwargs): + """ + This method receives a process chain name for an actinia module and + kwargs to fill the template values. It loads the according process + chain template from kvdb or filesystem. The received values will be + replaced to be passed to actinia. In case the template has more + placeholder values than it receives, the missing attribute is + returned as string. + """ + + pc = actiniamodulename + tpl_source = "" + + # first see if a user template exists + tpl = get_user_template(pc) + tpl_source = get_user_template_source(pc) + if tpl is False: + # then fall back to global filesystem template + tpl = get_global_template(pc) + tpl_source = get_global_template_source(pc) + + undef = get_template_undef(tpl_source) + parsed_content = pcTplEnv.parse(tpl_source) + + fill_env_values(kwargs, undef) + + errors = check_for_errors(undef, parsed_content, tpl_source, kwargs) + if errors is not None: + return errors + + pc_template = json.loads(tpl.render(**kwargs).replace("\n", "")) + return pc_template["template"] diff --git a/src/actinia_module_plugin/core/modules/actinia_common.py b/src/actinia_module_plugin/core/modules/actinia_common.py index 2d8f935..89c1bd8 100644 --- a/src/actinia_module_plugin/core/modules/actinia_common.py +++ b/src/actinia_module_plugin/core/modules/actinia_common.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-2021 mundialis GmbH & Co. KG +Copyright (c) 2018-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,19 +22,19 @@ __license__ = "Apache-2.0" __author__ = "Carmen Tawalika, Anika Weinmann" -__copyright__ = "Copyright 2019-2023, mundialis" +__copyright__ = "Copyright 2019-2025, mundialis" __maintainer__ = "Carmen Tawalika" import json import re -from os import environ as env # from actinia_module_plugin.core.common import filter_func from actinia_module_plugin.core.modules.actinia_global_templates import ( createProcessChainTemplateListFromFileSystem, ) from actinia_module_plugin.core.common import ( + ENV, get_user_template, get_user_template_source, get_global_template, @@ -49,13 +49,6 @@ from actinia_module_plugin.model.modules import Module -ENV = { - key.replace("TEMPLATE_VALUE_", ""): val - for key, val in env.items() - if key.startswith("TEMPLATE_VALUE_") -} - - def render_template(pc, return_source=False): # first see if a user template exists tpl = get_user_template(pc) @@ -531,10 +524,15 @@ def createActiniaModule(resourceBaseSelf, processchain): else: categories.append("global-template") + projects = [] + if "projects" in pc_template: + projects = pc_template["projects"] + virtual_module = Module( id=pc_template["id"], description=pc_template["description"], categories=categories, + projects=projects, parameters=pt.vm_params, returns=pt.vm_returns, ) diff --git a/src/actinia_module_plugin/core/processing.py b/src/actinia_module_plugin/core/processing.py index 84f9345..baaaafc 100644 --- a/src/actinia_module_plugin/core/processing.py +++ b/src/actinia_module_plugin/core/processing.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-2021 mundialis GmbH & Co. KG +Copyright (c) 2018-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,27 +21,10 @@ __license__ = "Apache-2.0" __author__ = "Carmen Tawalika" -__copyright__ = "Copyright 2019, mundialis" +__copyright__ = "Copyright 2019-2025, mundialis" __maintainer__ = "Carmen Tawalika" -import json - -from actinia_module_plugin.core.common import ( - get_user_template, - get_user_template_source, - get_global_template, - get_global_template_source, -) -from actinia_module_plugin.core.template_parameters import ( - get_not_needed_params, - get_template_undef, -) -from actinia_module_plugin.core.modules.actinia_common import ENV -from actinia_module_plugin.resources.logging import log -from actinia_module_plugin.resources.templating import pcTplEnv - - def build_kwargs_for_template_rendering(module): """ This method receives a process chain for an actinia module, isolates @@ -65,68 +48,3 @@ def build_kwargs_for_template_rendering(module): kwargs[key] = val return kwargs - - -def check_for_errors(undef, parsed_content, tpl_source, kwargs): - """ - This method checks if all placeholders are filled with values and - returns the placeholder if missing. Exceptions are default values for which - the given default value can be used and if statements for which the value - can be empty. - """ - # find default variables from processchain and variables which are only in - # an if statement and has not to be set - not_needed_vars = get_not_needed_params(undef, tpl_source, parsed_content) - - for i in undef: - # check if undef variables are needed or set in the kwargs - if i not in kwargs.keys() and i not in not_needed_vars: - log.error('Required parameter "' + i + '" not in process chain!') - return i - - return None - - -def fill_env_values(filled_params, undef): - """ - This function checks if a undefined variable is set in the environment - variables and set it in kwargs if not already set. - """ - if len(ENV) > 0: - for param in undef: - if param not in filled_params and param.upper() in ENV: - filled_params[param] = ENV[param.upper()] - - -def fillTemplateFromProcessChain(module): - """ - This method receives a process chain for an actinia module and loads - the according process chain template from kvdb or filesystem. The - received values will be replaced to be passed to actinia. In case the - template has more placeholder values than it receives, the missing - attribute is returned as string. - """ - - kwargs = build_kwargs_for_template_rendering(module) - tpl_source = "" - pc = module["module"] - - # first see if a user template exists - tpl = get_user_template(pc) - tpl_source = get_user_template_source(pc) - if tpl is False: - # then fall back to global filesystem template - tpl = get_global_template(pc) - tpl_source = get_global_template_source(pc) - - undef = get_template_undef(tpl_source) - parsed_content = pcTplEnv.parse(tpl_source) - - fill_env_values(kwargs, undef) - - errors = check_for_errors(undef, parsed_content, tpl_source, kwargs) - if errors is not None: - return errors - - pc_template = json.loads(tpl.render(**kwargs).replace("\n", "")) - return pc_template["template"]["list"] diff --git a/src/actinia_module_plugin/endpoints.py b/src/actinia_module_plugin/endpoints.py index c31595a..a60f944 100644 --- a/src/actinia_module_plugin/endpoints.py +++ b/src/actinia_module_plugin/endpoints.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2018-2021 mundialis GmbH & Co. KG +Copyright (c) 2018-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ """ __author__ = "Carmen Tawalika, Anika Weinmann" -__copyright__ = "2018-2024 mundialis GmbH & Co. KG" +__copyright__ = "2018-2025 mundialis GmbH & Co. KG" __license__ = "Apache-2.0" @@ -35,6 +35,9 @@ from actinia_module_plugin.api.modules.actinia import ( DescribeProcessChainTemplate, ) +from actinia_module_plugin.api.modules.actinia_process import ( + ProcessActiniaModule, +) from actinia_module_plugin.api.modules.combined import ListVirtualModules from actinia_module_plugin.api.modules.combined import DescribeVirtualModule from actinia_module_plugin.api.processing import ( @@ -102,6 +105,9 @@ def create_endpoints(flask_api): apidoc.add_resource( DescribeProcessChainTemplate, "/actinia_modules/" ) + apidoc.add_resource( + ProcessActiniaModule, "/actinia_modules//process" + ) apidoc.add_resource(ListVirtualModules, "/modules") apidoc.add_resource(DescribeVirtualModule, "/modules/") diff --git a/src/actinia_module_plugin/model/modules.py b/src/actinia_module_plugin/model/modules.py index d26e8e6..5bedd8b 100644 --- a/src/actinia_module_plugin/model/modules.py +++ b/src/actinia_module_plugin/model/modules.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Copyright (c) 2019-present mundialis GmbH & Co. KG +Copyright (c) 2019-2025 mundialis GmbH & Co. KG Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ """ __author__ = "Anika Bettge, Carmen Tawalika" -__copyright__ = "2019-present mundialis GmbH & Co. KG" +__copyright__ = "2019-2025 mundialis GmbH & Co. KG" __license__ = "Apache-2.0" @@ -137,6 +137,12 @@ class Module(Schema): 'category "grass-module" and the actinia core ' 'modules are identified with "actinia-module"', }, + "projects": { + "type": "array", + "items": {"type": "string"}, + "description": "A comma separated list of GRASS GIS projects for " + "which the process is suitable.", + }, "parameters": ModuleParameter, "returns": ModuleReturns, "import_descr": ModuleImportDescription, diff --git a/src/actinia_module_plugin/resources/config.py b/src/actinia_module_plugin/resources/config.py index 5077819..218ad72 100644 --- a/src/actinia_module_plugin/resources/config.py +++ b/src/actinia_module_plugin/resources/config.py @@ -59,18 +59,19 @@ def __init__(self): named DEFAULT_CONFIG_PATH/**/*.ini exist. On first import of the module it is initialized. """ + from actinia_module_plugin.resources.logging import log config = configparser.ConfigParser() config.read(CONFIG_FILES) if len(config) <= 1: - print("Could not find any config file, using default values.") + log.info("Could not find any config file, using default values.") return - print("Loading config files: " + str(CONFIG_FILES) + " ...") + log.info("Loading config files: " + str(CONFIG_FILES) + " ...") with open(GENERATED_CONFIG, "w") as configfile: config.write(configfile) - print("Configuration written to " + GENERATED_CONFIG) + log.debug("Configuration written to " + GENERATED_CONFIG) # LOGGING if config.has_section("LOGCONFIG"): diff --git a/src/actinia_module_plugin/resources/logging.py b/src/actinia_module_plugin/resources/logging.py index 5164206..8041e7b 100644 --- a/src/actinia_module_plugin/resources/logging.py +++ b/src/actinia_module_plugin/resources/logging.py @@ -116,6 +116,8 @@ def createGunicornLogger(): logging.getLogger(name).handlers = [] +if not werkzeugLog: + createWerkzeugLogger() +if not gunicornLog: + createGunicornLogger() createLogger() -createWerkzeugLogger() -createGunicornLogger() diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..6b73253 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,38 @@ +# Tests + +## Running tests locally + +When writing new tests it is useful to run selected tests locally and isolated. In this way it is also possible to debug interactively: + +1. In the `docker/actinia-module-plugin-test/Dockerfile` comment out `RUN ./tests_with_kvdb.sh` (last line) + +1. Then run `docker build`: + +```bash +docker build -f docker/actinia-module-plugin-test/Dockerfile -t actinia-module-plugin-test:alpine . +``` + +3. To run only a few tests you can mark the tests for development with + `@pytest.mark.dev` and add `import pytest` to the `.py` file/s with the tests you want to run. + (For best practice examples on the use of pytest-decorators, see actinia-core `tests/unittests/test_version.py`) + +1. Start the docker container and mount your `tests` folder: + +```bash +docker run --rm -it --entrypoint sh -v `pwd`/tests:/src/actinia-module-plugin/tests actinia-module-plugin-test:alpine +# If you are not developing the tests you can run tests using the following command: +docker run --rm -it --entrypoint sh actinia-module-plugin-test:alpine +``` + +And then start the kvdb and run tests: +```bash +valkey-server & +sleep 1 +valkey-cli ping + +# run all tests +pytest + +# run only dev tests in debugger mode +pytest --pdb -x -m dev +``` diff --git a/tests/resources/actinia_modules/point_in_polygon.json b/tests/resources/actinia_modules/point_in_polygon.json index 9afe501..11996cc 100644 --- a/tests/resources/actinia_modules/point_in_polygon.json +++ b/tests/resources/actinia_modules/point_in_polygon.json @@ -14,5 +14,8 @@ } } ], + "projects": [ + "nc_spm_08" + ], "returns": [] } diff --git a/tests/resources/actinia_templates/user_point_in_polygon.json b/tests/resources/actinia_templates/user_point_in_polygon.json index 3d69816..b510481 100644 --- a/tests/resources/actinia_templates/user_point_in_polygon.json +++ b/tests/resources/actinia_templates/user_point_in_polygon.json @@ -1,5 +1,6 @@ { "id": "user_point_in_polygon", + "projects": ["nc_spm_08"], "description": "Imports point and polygon and checks if point is in polygon.", "template": { "list": [ diff --git a/tests/resources/processing/user_point_in_polygon_actinia_module_process.json b/tests/resources/processing/user_point_in_polygon_actinia_module_process.json new file mode 100644 index 0000000..c36aa46 --- /dev/null +++ b/tests/resources/processing/user_point_in_polygon_actinia_module_process.json @@ -0,0 +1 @@ +"https://raw.githubusercontent.com/mmacata/pagestest/gh-pages/pointInBonn.geojson" diff --git a/tests/test_modules_actinia_processing.py b/tests/test_modules_actinia_processing.py new file mode 100644 index 0000000..8732bf9 --- /dev/null +++ b/tests/test_modules_actinia_processing.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2025 mundialis GmbH & Co. KG + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Test Module Lists and Self-Description +""" + +__license__ = "Apache-2.0" +__author__ = "Carmen Tawalika" +__copyright__ = "Copyright 2025, mundialis" + +import json +from flask import Response + +from actinia_api import URL_PREFIX + +from testsuite import ( + ActiniaTestCase, + check_started_process, + delete_user_template, + import_user_template, +) + + +class ActiniaModulesProcessingTest(ActiniaTestCase): + def test_process_global_module(self): + """ + Test HTTP POST /actinia_modules//processing of + global templates ephemeral processing + """ + respStatusCode = 200 + json_path = "tests/resources/processing/global_point_in_polygon.json" + url_path = "/actinia_modules/point_in_polygon/process" + + with open(json_path) as file: + pc_template = json.load(file) + + resp = self.app.post( + URL_PREFIX + url_path, + headers=self.user_auth_header, + data=json.dumps(pc_template), + content_type="application/json", + ) + + assert isinstance(resp, Response) + assert resp.status_code == respStatusCode + assert hasattr(resp, "json") + + check_started_process(self, resp) + + def test_process_user_module(self): + """ + Test HTTP POST /actinia_modules//processing of + user templates ephemeral processing + """ + import_user_template(self, "user_point_in_polygon") + + respStatusCode = 200 + json_path = ( + "tests/resources/processing/" + "user_point_in_polygon_actinia_module_process.json" + ) + url_path = "/actinia_modules/user_point_in_polygon/process" + + with open(json_path) as file: + pc_template = json.load(file) + + resp = self.app.post( + URL_PREFIX + url_path, + headers=self.user_auth_header, + data=json.dumps(pc_template), + content_type="application/json", + ) + + assert isinstance(resp, Response) + assert resp.status_code == respStatusCode + assert hasattr(resp, "json") + + check_started_process(self, resp) + + delete_user_template(self, "user_point_in_polygon")