Use of new taxonomy tools and introduction of postinstall django app commands management

.github/workflows/engine_pr_test.yml

@@ -160,18 +160,18 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Actualize 'aelo' templates for email notifications
+        run: |
+          for file in openquake/server/templates/registration/*.aelo.tmpl; do
+              cp -- "$file" "${file%.aelo.tmpl}"
+          done
       - name: Install dependencies
         run: |
           if [[ $FROM_FORK == "true" ]]; then
             python install.py devel --version=$GITHUB_HEAD_REF --from_fork
           else
             python install.py devel --version=$GITHUB_HEAD_REF
           fi
-      - name: Actualize 'aelo' templates for email notifications
-        run: |
-          for file in openquake/server/templates/registration/*.aelo.tmpl; do
-              cp -- "$file" "${file%.aelo.tmpl}"
-          done
       - name: Server 'AELO' mode tests
         run: |
           source ~/openquake/bin/activate
@@ -199,18 +199,18 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Actualize 'impact' templates for email notifications
+        run: |
+          for file in openquake/server/templates/registration/*.impact.tmpl; do
+              cp -- "$file" "${file%.impact.tmpl}"
+          done
       - name: Install dependencies
         run: |
           if [[ $FROM_FORK == "true" ]]; then
             python install.py devel --version=$GITHUB_HEAD_REF --from_fork
           else
             python install.py devel --version=$GITHUB_HEAD_REF
           fi
-      - name: Actualize 'impact' templates for email notifications
-        run: |
-          for file in openquake/server/templates/registration/*.impact.tmpl; do
-              cp -- "$file" "${file%.impact.tmpl}"
-          done
       - name: Server 'ARISTOTLE' mode tests
         run: |
           set -x

install.py

@@ -210,7 +210,7 @@ class devel(user):
 }
 DEMOS = "https://artifacts.openquake.org/travis/demos-master.zip"
 GITBRANCH = "https://github.com/gem/oq-engine/archive/%s.zip"
-URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/latest/"
+URL_STANDALONE = "https://wheelhouse.openquake.org/py/standalone/engine-3.23/"
 
 
 def ensure(pip=None, pyvenv=None):
@@ -263,37 +263,75 @@ def get_requirements_branch(version, inst, from_fork):
         return version
 
 
-def install_standalone(venv):
+def install_or_postinstall_standalone(venv, is_install=True):
     """
-    Install the standalone Django applications if possible
+    Install the standalone Django applications if possible or
+    run '<app>_postinstall' command if it exists
     """
     errors = []
-    print("The standalone applications are not installed yet")
+    if is_install:
+        print("The standalone applications are not installed yet")
+    else:
+        print("Run '<app>_postinstall' command for each standalone\n"
+              " Django applications, if it exists")
     if sys.platform == "win32":
         if os.path.exists("python\\python._pth.old"):
             pycmd = inst.VENV + "\\python.exe"
         else:
             pycmd = inst.VENV + "\\Scripts\\python.exe"
     else:
         pycmd = inst.VENV + "/bin/python3"
-    for app in [
-        "oq-platform-standalone",
-        "oq-platform-ipt",
-        "oq-platform-taxonomy",
-    ]:
-        try:
-            print("Applications " + app + " are not installed yet \n")
 
-            subprocess.check_call(
-                [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE,
-                 app]
-            )
-        except Exception as exc:
-            # for instance is somebody removed a wheel from the wheelhouse
-            errors.append("%s: could not install %s" % (exc, app))
+    STANDALONE_APP_INFO = [
+        {"pkg": "oq-platform-standalone", "name": None},
+        {"pkg": "oq-platform-ipt",        "name": "openquakeplatform_ipt"},
+        {"pkg": "oq-platform-taxonomy",   "name": "openquakeplatform_taxonomy"},
+        {"pkg": "django-gem-taxonomy",    "name": "django_gem_taxonomy"},
+    ]
+
+    if is_install:
+        for app in STANDALONE_APP_INFO:
+            try:
+                print("Applications " + app['pkg'] + " are not installed yet \n")
+
+                subprocess.check_call(
+                    [pycmd, "-m", "pip", "install", "--find-links", URL_STANDALONE,
+                     app['pkg']]
+                )
+            except Exception as exc:
+                # for instance is somebody removed a wheel from the wheelhouse
+                errors.append("%s: could not install %s" % (exc, app['pkg']))
+    else:
+        for app in STANDALONE_APP_INFO:
+            if not app['name']:
+                continue
+
+            try:
+                if sys.platform == "win32":
+                    django_admin = ['Scripts', 'django-admin.exe']
+                else:
+                    django_admin = ["bin", "django-admin"]
+
+                django_env = os.environ.copy()
+                django_env["DJANGO_SETTINGS_MODULE"] = "openquake.server.settings"
+
+                subprocess.check_call(
+                    [os.path.join(inst.VENV, *django_admin),
+                     "openquake_engine_postinstall", app['name']],
+                    env=django_env)
+            except Exception as exc:
+                # for instance is somebody removed a wheel from the wheelhouse
+                errors.append("%s: error during %s postinstall command execution" % (exc, app['name']))
+
     return errors
 
 
+def install_standalone(venv):
+    return install_or_postinstall_standalone(venv, is_install=True)
+
+def postinstall_standalone(venv):
+    return install_or_postinstall_standalone(venv, is_install=False)
+
 def before_checks(inst, venv, port, remove, usage):
     """
     Checks to perform before the installation
@@ -386,7 +424,7 @@ def fix_version(commit, venv):
         f.write("".join(lines))
 
 
-def install(inst, version, from_fork):
+def install(inst, version, from_fork, keepvenv):
     """
     Install the engine in one of the three possible modes
     """
@@ -406,9 +444,10 @@ def install(inst, version, from_fork):
         if inst is server or inst is devel_server:
             subprocess.check_call(["chown", "openquake", inst.OQDATA])
 
-    # recreate the openquake venv
-    ensure(pyvenv=inst.VENV)
-    print("Created %s" % inst.VENV)
+    if not keepvenv:
+        # recreate the openquake venv
+        ensure(pyvenv=inst.VENV)
+        print("Created %s" % inst.VENV)
 
     if sys.platform == "win32":
         if os.path.exists("python\\python._pth.old"):
@@ -506,6 +545,8 @@ def install(inst, version, from_fork):
     if inst in (user, devel):  # create/upgrade the db in the default location
         subprocess.run([oqreal, "engine", "--upgrade-db"])
 
+    errors += postinstall_standalone(inst.VENV)
+
     if (
         inst is server
         and not os.path.exists(inst.OQ)
@@ -610,6 +651,8 @@ def remove(inst):
         help="the kind of installation you want",
     )
     parser.add_argument("--venv", help="venv directory")
+    parser.add_argument("--keepvenv", action="store_false",
+                        help="keep the current virtual environment")
     parser.add_argument("--remove", action="store_true",
                         help="disinstall the engine")
     parser.add_argument("--version", help="version to install (default stable)")
@@ -628,7 +671,7 @@ def remove(inst):
         if args.remove:
             remove(inst)
         else:
-            errors = install(inst, args.version, args.from_fork)
+            errors = install(inst, args.version, args.from_fork, args.keepvenv)
             if errors:
                 # NB: even if one of the tools is missing, the engine will work
                 sys.exit('\n'.join(errors))

openquake/server/management/commands/openquake_engine_postinstall.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+#
+# oq-geoviewer
+# Copyright (C) 2018-2019 GEM Foundation
+#
+# oq-geoviewer is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# oq-geoviewer is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# import subprocess
+from django.apps import apps as django_apps
+from django.core.management import call_command, get_commands
+import sys
+from django.core.management.base import BaseCommand
+
+class Command(BaseCommand):
+    help = ("Command that run a '<app_name>_postinstall' command if it exists")
+
+    def add_arguments(self, parser):
+        parser.add_argument('django_app',
+                            help='django application name')
+
+    def handle(self, *args, **options):
+        found = False
+        for app in django_apps.get_app_configs():
+            label = app.label
+            if options['django_app'] == label:
+                found = True
+                break
+
+        if not found:
+            self.stdout.write(
+                self.style.ERROR(
+                    "No django app '%s' found." % (options['django_app'],))
+                )
+            sys.exit(1)
+
+        postinstall_cmd = options['django_app'] + '_postinstall'
+        django_cmds = get_commands()
+        if postinstall_cmd not in django_cmds:
+            self.stdout.write(
+                self.style.WARNING(
+                    "No 'postinst' action needed for app %s, skipped." % (options['django_app'],))
+                )
+            sys.exit(0)
+
+        call_command(postinstall_cmd)

openquake/server/settings.py

@@ -26,8 +26,6 @@
 from openquake.baselib import config
 from openquake.commonlib import datastore
 
-# optionally overrided in local_settings.py
-STANDALONE_APP_NAME_MAP = {}
 try:
     from openquakeplatform.settings import STANDALONE, STANDALONE_APPS
 except ImportError:
@@ -213,6 +211,14 @@
 
 APPLICATION_MODE = 'PUBLIC'
 
+# Definition of Django applications
+STANDALONE_APP_NAME_MAP = {
+        'openquakeplatform_ipt': 'ipt',
+        'django_gem_taxonomy': 'taxonomy',
+    }
+if APPLICATION_MODE != 'TOOLS_ONLY':
+    STANDALONE_APP_NAME_MAP['openquakeplatform_taxonomy'] = 'glossary'
+
 ARISTOTLE_DEFAULT_USGS_ID = 'us7000n7n8'  # loadable and convertible rupture
 # ARISTOTLE_DEFAULT_USGS_ID = 'us6000jllz'  # loadable but with conversion err
 

pytest.ini

@@ -2,4 +2,5 @@
 markers =
   slow: a slow test
 addopts = --tb short
+consider_namespace_packages = true
 DJANGO_SETTINGS_MODULE = openquake.server.settings

setup.py

@@ -19,7 +19,8 @@
 import os
 import re
 import sys
-from setuptools import setup, find_packages
+from setuptools import setup, find_namespace_packages
+
 
 if sys.version_info < (3, 9):
     sys.exit('Sorry, Python < 3.9 is not supported')
@@ -112,11 +113,16 @@ def get_readme():
         'Environment :: Console',
         'Environment :: Web Environment',
     ],
-    packages=find_packages(exclude=["qa_tests", "qa_tests.*",
-                                    "tools",
-                                    "*.*.tests", "*.*.tests.*",
-                                    "openquake.engine.bin",
-                                    "openquake.engine.bin.*"]),
+    packages=find_namespace_packages(include=[
+                                         "openquake.*",
+                                     ],
+                                     exclude=[
+                                         "qa_tests", "qa_tests.*",
+                                         "tools",
+                                         "*.*.tests", "*.*.tests.*",
+                                         "openquake.engine.bin",
+                                         "openquake.engine.bin.*",
+                                     ]),
     py_modules=PY_MODULES,
     include_package_data=True,
     package_data={"openquake.engine": [