WIP - manage satellite Django apps for post-installation procedures

.github/workflows/webui_rh_test.yml

@@ -66,25 +66,32 @@ jobs:
         eval '$mypython -m pip install -e . '
         ## Standalone apps
         echo "Downloading standalone apps"
-        for app in oq-platform-standalone oq-platform-ipt oq-platform-taxonomy; do
-         # do not fail on exit code 2 if the branch
-         # does not exist in the app repository
-         set +e
-         git ls-remote --exit-code --heads https://github.com/gem/${app}.git $BRANCH >/dev/null 2>&1
-         EXIT_CODE=$?
-         # set again to fail on exit code not 0
-         set -e
-         if [[ $EXIT_CODE == '0' ]]; then
-            echo "Git branch '$BRANCH' exists in the remote repository"
-            TOOLS_BRANCH=$BRANCH
-         elif [[ $EXIT_CODE == '2' ]]; then
-            echo "Git branch '$BRANCH' does not exist in the remote repository"
-            TOOLS_BRANCH=master
-         fi
-         echo "We need to use the branch $TOOLS_BRANCH for the standalone apps"
-         git clone -b ${TOOLS_BRANCH} --depth=1 https://github.com/gem/${app}.git
-         eval '$mypython -m pip install -e ./${app}'
+        for app in oq-platform-standalone django-gem-taxonomy oq-platform-ipt oq-platform-taxonomy; do
+            for branch in "$BRANCH" "master" "main"; do
+                TOOLS_BRANCH="$branch"
+                # do not fail on exit code 2 if the branch
+                # does not exist in the app repository
+                set +e
+                git ls-remote --exit-code --heads https://github.com/gem/${app}.git $TOOLS_BRANCH >/dev/null 2>&1
+                EXIT_CODE=$?
+                # set again to fail on exit code not 0
+                set -e
+                if [[ $EXIT_CODE -eq 0 ]]; then
+                    echo "Git branch '$BRANCH' exists in the remote repository"
+                    break
+                fi
+            done
+            if [[ $EXIT_CODE -ne 0 ]]; then
+                break
+            fi
+            echo "We need to use the branch $TOOLS_BRANCH for the standalone apps"
+            git clone -b ${TOOLS_BRANCH} --depth=1 https://github.com/gem/${app}.git
+            eval '$mypython -m pip install -e ./${app}'
         done
+        if [ $EXIT_CODE -ne 0 ]; then
+            echo "No '$branch', nor 'master' or 'main' branch found for '$app' django app; failed"
+            exit 1
+        fi
         deactivate
     - name: Actualize 'default' templates for email notifications
       run: |

install.py

@@ -219,7 +219,9 @@ class devel(user):
     "win32": ("win64",),
 }
 GITBRANCH = "https://github.com/gem/oq-engine/archive/%s.zip"
-URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/"
+# FIXME just for devel test
+# URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/"
+URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/post-inst/"
 
 
 def ensure(pip=None, pyvenv=None):
@@ -271,37 +273,75 @@ def get_requirements_branch(version, inst, from_fork):
         return version
 
 
-def install_standalone(venv):
+def install_or_postinstall_standalone(venv, is_install=True):
     """
-    Install the standalone Django applications if possible
+    Install the standalone Django applications if possible or
+    run '<app>_postinstall' command if it exists
     """
     errors = []
-    print("The standalone applications are not installed yet")
+    if is_install:
+        print("The standalone applications are not installed yet")
+    else:
+        print("Run '<app>_postinstall' command for each standalone\n"
+              " Django applications, if it exists")
     if sys.platform == "win32":
         if os.path.exists("python\\python._pth.old"):
             pycmd = inst.VENV + "\\python.exe"
         else:
             pycmd = inst.VENV + "\\Scripts\\python.exe"
     else:
         pycmd = inst.VENV + "/bin/python3"
-    for app in [
-        "oq-platform-standalone",
-        "oq-platform-ipt",
-        "oq-platform-taxonomy",
-    ]:
-        try:
-            print("Applications " + app + " are not installed yet \n")
 
-            subprocess.check_call(
-                [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE,
-                 app]
-            )
-        except Exception as exc:
-            # for instance is somebody removed a wheel from the wheelhouse
-            errors.append("%s: could not install %s" % (exc, app))
+    STANDALONE_APP_INFO = [
+        {"pkg": "oq-platform-standalone", "name": None},
+        {"pkg": "oq-platform-ipt",        "name": "openquakeplatform_ipt"},
+        {"pkg": "oq-platform-taxonomy",   "name": "openquakeplatform_taxonomy"},
+        {"pkg": "django-gem-taxonomy",    "name": "django_gem_taxonomy"},
+    ]
+
+    if is_install:
+        for app in STANDALONE_APP_INFO:
+            try:
+                print("Applications " + app['pkg'] + " are not installed yet \n")
+
+                subprocess.check_call(
+                    [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE,
+                     app['pkg']]
+                )
+            except Exception as exc:
+                # for instance is somebody removed a wheel from the wheelhouse
+                errors.append("%s: could not install %s" % (exc, app['pkg']))
+    else:
+        for app in STANDALONE_APP_INFO:
+            if not app['name']:
+                continue
+
+            try:
+                if sys.platform == "win32":
+                    django_admin = ['Scripts', 'django-admin.exe']
+                else:
+                    django_admin = ["bin", "django-admin"]
+
+                django_env = os.environ.copy()
+                django_env["DJANGO_SETTINGS_MODULE"] = "openquake.server.settings"
+
+                subprocess.check_call(
+                    [os.path.join(inst.VENV, *django_admin),
+                     "openquake_engine_postinstall", app['name']],
+                    env=django_env)
+            except Exception as exc:
+                # for instance is somebody removed a wheel from the wheelhouse
+                errors.append("%s: error during %s postinstall command execution" % (exc, app['name']))
+
     return errors
 
 
+def install_standalone(venv):
+    return install_or_postinstall_standalone(venv, is_install=True)
+
+def postinstall_standalone(venv):
+    return install_or_postinstall_standalone(venv, is_install=False)
+
 def before_checks(inst, args, usage):
     """
     Checks to perform before the installation
@@ -311,6 +351,10 @@ def before_checks(inst, args, usage):
     if args.dbport:
         inst.DBPORT = int(args.dbport)
 
+    if args.novenv:
+        inst.VENV = os.path.join(os.getenv('LocalAppData'), 'Programs',
+                                 'OpenQuake Engine', 'python3')
+
     # check platform
     if (inst is server and sys.platform != "linux") or (
         inst is devel_server and sys.platform != "linux"
@@ -427,7 +471,7 @@ def normalize_version(version):
         return f"=={version}"
 
 
-def install(inst, version, from_fork):
+def install(inst, version, from_fork, novenv, noupgrade):
     """
     Install the engine in one of the three possible modes
     """
@@ -447,34 +491,38 @@ def install(inst, version, from_fork):
         if inst is server or inst is devel_server:
             subprocess.check_call(["chown", "openquake", inst.OQDATA])
 
-    # recreate the openquake venv
-    ensure(pyvenv=inst.VENV)
-    print("Created %s" % inst.VENV)
+    if not novenv:
+        # recreate the openquake venv
+        ensure(pyvenv=inst.VENV)
+        print("Created %s" % inst.VENV)
 
-    if sys.platform == "win32":
-        if os.path.exists("python\\python._pth.old"):
-            pycmd = inst.VENV + "\\python.exe"
+        if sys.platform == "win32":
+            if os.path.exists("python\\python._pth.old"):
+                pycmd = inst.VENV + "\\python.exe"
+            else:
+                pycmd = inst.VENV + "\\Scripts\\python.exe"
         else:
-            pycmd = inst.VENV + "\\Scripts\\python.exe"
+            pycmd = inst.VENV + "/bin/python3"
     else:
-        pycmd = inst.VENV + "/bin/python3"
+        pycmd = os.path.join(inst.VENV, 'python.exe')
 
     # upgrade pip and before check that it is installed in venv
     if sys.platform != "win32":
         ensure(pip=pycmd)
-        subprocess.check_call(
-            [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel"]
-        )
+        subprocess.check_call([pycmd, "-m", "pip", "install"] + ([
+        ] if noupgrade else ["--upgrade"]) + [
+            "pip", "wheel"])
     else:
         if os.path.exists("python\\python._pth.old"):
-            subprocess.check_call(
-                [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel",
-                 "urllib3"])
+            subprocess.check_call([pycmd, "-m", "pip", "install"] + ([
+            ] if noupgrade else ["--upgrade"]) + [
+                "pip", "wheel", "urllib3"])
         else:
-            subprocess.check_call([pycmd, "-m", "ensurepip", "--upgrade"])
-            subprocess.check_call(
-                [pycmd, "-m", "pip", "install", "--upgrade", "pip", "wheel",
-                 "urllib3"])
+            subprocess.check_call([pycmd, "-m", "ensurepip"] + ([
+            ] if noupgrade else ["--upgrade"]))
+            subprocess.check_call([pycmd, "-m", "pip", "install"] + ([
+            ] if noupgrade else ["--upgrade"]) + [
+                "pip", "wheel", "urllib3"])
 
     # install the requirements
     branch = get_requirements_branch(version, inst, from_fork)
@@ -506,12 +554,14 @@ def install(inst, version, from_fork):
         subprocess.check_call([pycmd, "-m", "pip", "install", "-e", CDIR])
     elif version is None:  # install the stable version
         subprocess.check_call(
-            [pycmd, "-m", "pip", "install", "--upgrade", "openquake.engine"]
+            [pycmd, "-m", "pip", "install"] + ([
+                ] if noupgrade else ["--upgrade"]) + ["openquake.engine"]
         )
     elif re.match(r"\d+(\.\d+)+", version):  # install an official version
         subprocess.check_call(
-            [pycmd, "-m", "pip", "install", "--upgrade",
-             f"openquake.engine{normalize_version(version)}"]
+            [pycmd, "-m", "pip", "install"] + ([] if noupgrade
+                                               else ["--upgrade"]) +
+            [f"openquake.engine{normalize_version(version)}"]
         )
     else:  # install a branch from github (only for user or server)
         commit = latest_commit(version)
@@ -521,8 +571,9 @@ def install(inst, version, from_fork):
             custom_env["TMPDIR"] = tmp  # Linux/macOS
             custom_env["TEMP"] = tmp    # Windows
             subprocess.check_call(
-                [pycmd, "-m", "pip", "install",
-                 "--upgrade", GITBRANCH % commit, "--no-clean"],
+                [pycmd, "-m", "pip", "install"] + (
+                    [] if noupgrade else ["--upgrade"]) + [GITBRANCH % commit,
+                                                           "--no-clean"],
                 env=custom_env)
         fix_version(commit, inst.VENV)
 
@@ -553,6 +604,8 @@ def install(inst, version, from_fork):
     if inst in (user, devel):  # create/upgrade the db in the default location
         subprocess.run([oqreal, "engine", "--upgrade-db"])
 
+    errors += postinstall_standalone(inst.VENV)
+
     if (
         inst is server
         and not os.path.exists(inst.OQ)
@@ -646,6 +699,10 @@ def remove(inst):
         help="the kind of installation you want",
     )
     parser.add_argument("--venv", help="venv directory")
+    parser.add_argument("--novenv", action="store_true",
+                        help="keep the current python environment")
+    parser.add_argument("--noupgrade", action="store_true",
+                        help="not use '--upgrade' in pip install calls")
     parser.add_argument("--remove", action="store_true",
                         help="disinstall the engine")
     parser.add_argument("--version", help="version to install (default stable)")
@@ -663,7 +720,7 @@ def remove(inst):
         if args.remove:
             remove(inst)
         else:
-            errors = install(inst, args.version, args.from_fork)
+            errors = install(inst, args.version, args.from_fork, args.novenv, args.noupgrade)
             if errors:
                 # NB: even if one of the tools is missing, the engine will work
                 sys.exit('\n'.join(errors))

openquake/server/management/commands/openquake_engine_postinstall.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# oq-geoviewer
+# Copyright (C) 2018-2019 GEM Foundation
+#
+# oq-geoviewer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# oq-geoviewer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# import subprocess
+from django.apps import apps as django_apps
+from django.core.management import call_command, get_commands
+import sys
+from django.core.management.base import BaseCommand
+
+class Command(BaseCommand):
+    help = ("Command that run a '<app_name>_postinstall' command if it exists")
+
+    def add_arguments(self, parser):
+        parser.add_argument('django_app',
+                            help='django application name')
+
+    def handle(self, *args, **options):
+        found = False
+        for app in django_apps.get_app_configs():
+            label = app.label
+            if options['django_app'] == label:
+                found = True
+                break
+
+        if not found:
+            self.stdout.write(
+                self.style.ERROR(
+                    "No django app '%s' found." % (options['django_app'],))
+                )
+            sys.exit(1)
+
+        postinstall_cmd = options['django_app'] + '_postinstall'
+        django_cmds = get_commands()
+        if postinstall_cmd not in django_cmds:
+            self.stdout.write(
+                self.style.WARNING(
+                    "No 'postinst' action needed for app %s, skipped." % (options['django_app'],))
+                )
+            sys.exit(0)
+
+        call_command(postinstall_cmd)

openquake/server/settings.py

@@ -230,6 +230,14 @@
 # IMPACT_DEFAULT_USGS_ID = 'us7000n7n8'  # loadable and convertible rupture
 # IMPACT_DEFAULT_USGS_ID = 'us6000jllz'  # loadable but with conversion err
 
+# Definition of Django applications
+STANDALONE_APP_NAME_MAP = {
+        'openquakeplatform_ipt': 'ipt',
+        'django_gem_taxonomy': 'taxonomy',
+    }
+if APPLICATION_MODE != 'TOOLS_ONLY':
+    STANDALONE_APP_NAME_MAP['openquakeplatform_taxonomy'] = 'glossary'
+
 EXTERNAL_TOOLS = os.environ.get('EXTERNAL_TOOLS', False) == 'True'
 
 # If False, a warning is displayed in case a newer version of the engine has

pyproject.toml

@@ -32,8 +32,26 @@ dependencies = [
     "django>=4.2, <5",
     "psutil",
     "requests",
-    "pyzmq",
+    "pyzmq~=26.0.3; python_version <= '3.11'",
+    "pyzmq~=26.2.0; python_version >  '3.11'",    
     "h3>=4.4",
+
+    # unplanned deps
+    "decorator~=5.1.1",
+    "pillow~=12.2.0",
+    "toml~=0.10.2",
+    "numba~=0.61.2",
+    "alpha_shapes~=1.1.0",
+    "docutils~=0.20.1",
+    "django_cors_headers~=4.3.1",
+
+    # geopackages
+    "gdal~=3.11.4; sys_platform == 'win32'",
+    "gdal~=3.13.1; sys_platform != 'win32'",
+    "pyproj~=3.7.2",
+    "geopandas~=1.1.3; sys_platform == 'win32'",
+    "geopandas~=1.1.4; sys_platform != 'win32'",
+    "fiona~=1.10.1",
 ]
 
 [project.urls]