[rel-db] Initial migration

.github/docker-compose/docker-compose.prod.yml

@@ -1,7 +1,10 @@
-version: "3"
 services:
     backend-action:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9002:9002"
@@ -14,7 +17,11 @@ services:
         secrets:
             - postgres_password
     backend-presenter:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9003:9003"

.github/workflows/continuous_integration.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v6
+      with:
+        submodules: recursive
 
     - name: Create secrets for datastore
       run: mkdir secrets && echo -n "openslides" > secrets/postgres_password
@@ -64,6 +66,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         path: openslides-backend
+        submodules: recursive
 
     - name: Set env vars
       working-directory: openslides-backend/requirements

Makefile

@@ -45,6 +45,11 @@ check-all: validate-models-yml check-models check-initial-data-json check-exampl
 
 # Models
 
+generate-schema:
+	make -C meta/dev generate-relational-schema
+
+generate-db: | generate-schema create-database-with-schema
+
 generate-models:
 	python cli/generate_models.py $(MODELS_PATH)
 	black openslides_backend/models/models.py

cli/generate_models.py

@@ -4,13 +4,7 @@
 from textwrap import dedent
 from typing import Any, Optional
 
-from cli.util.util import (
-    ROOT,
-    assert_equal,
-    open_output,
-    open_yml_file,
-    parse_arguments,
-)
+from cli.util.util import ROOT, assert_equal, open_output, parse_arguments
 from meta.dev.src.helper_get_names import (
     FieldSqlErrorType,
     HelperGetNames,
@@ -76,8 +70,6 @@
     """
 )
 
-MODELS: dict[str, dict[str, Any]] = {}
-
 
 def main() -> None:
     """
@@ -101,10 +93,8 @@ def main() -> None:
             to: some_model/some_attribute_id
     """
     args = parse_arguments(SOURCE)
-    global MODELS
-    MODELS = open_yml_file(args.filename)
 
-    InternalHelper.MODELS = MODELS
+    InternalHelper.read_models_yml(SOURCE)
 
     # Load and parse models.yml
     with open_output(DESTINATION, args.check) as dest:
@@ -114,7 +104,7 @@ def main() -> None:
             + ", ".join(mixin.__name__ for mixin in MODEL_MIXINS.values())
             + "\n"
         )
-        for collection, fields in MODELS.items():
+        for collection, fields in InternalHelper.MODELS.items():
             if collection.startswith("_"):
                 continue
             model = Model(collection, fields)
@@ -132,7 +122,7 @@ def get_model_field(collection: str, field_name: str) -> str | dict:
     Helper function the get a specific model field. Used to create generic relations.
     """
 
-    model = MODELS.get(collection)
+    model = InternalHelper.MODELS.get(collection)
     if model is None:
         raise ValueError(f"Collection {collection} does not exist.")
     value = model.get(field_name)
@@ -376,16 +366,21 @@ def get_view_field_state_write_fields(
                 else:
                     write_fields = (table_name, field1, field2, [])
             elif "generic-relation-list" in (field_type, foreign_type):
-                write_fields = self.get_write_fields_for_generic(own, foreign_fields)
+                write_fields = self.get_write_fields_for_generic(
+                    own, foreign_fields, primary
+                )
 
         assert error == "", error
 
         return is_view_field, primary, write_fields
 
     def get_write_fields_for_generic(
-        self, own: TableFieldType, foreign_fields: list[TableFieldType]
+        self, own: TableFieldType, foreign_fields: list[TableFieldType], primary: bool
     ) -> tuple[str, str, str, list[str]] | None:
-        table_name = HelperGetNames.get_gm_table_name(own)
+        if primary:
+            table_name = HelperGetNames.get_gm_table_name(own)
+        else:
+            table_name = HelperGetNames.get_gm_table_name(foreign_fields[0])
         field1 = f"{own.table}_{own.ref_column}"
         field2 = own.intermediate_column
         return (

data/example-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

data/initial-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

dev/entrypoint.sh

@@ -3,20 +3,19 @@
 printf "enter entrypoint.sh"
 set -e
 
-printf "\nOpenslides Database:\n"
+printf "\nOpenslides DBMS:\n"
 printf "Export env variables for database.\n"
 source scripts/export_database_variables.sh
 
 meta/dev/scripts/wait-for-database.sh
-printf "Database is started.\n"
+printf "DBMS is started.\n"
 
 printf "Create schema.\n"
 python cli/create_schema.py
 printf "\n"
 
-# TODO: Re-add this code
-# printf "\nMigrations:\n"
-# python openslides_backend/migrations/migrate.py finalize
-# printf "\n"
+printf "\nMigrations:\n"
+python openslides_backend/migrations/migrate.py finalize
+printf "\n"
 
 exec "$@"

docs/migration_route.md

@@ -3,20 +3,23 @@
 Migrations are available via the internal route `/internal/migrations/` with the following payload:
 ```js
 {
-    "cmd": "migrate" | "finalize" | "reset" | "clear-collectionfield-tables" | "stats" | "progress",
+    "cmd": "migrate" | "finalize" | "reset" | "stats",
     "verbose": bool
 }
 ```
 
-All commands except `progress` are directly translated the the respective method calls to the datastore module as well as the `verbose` flag. While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `progress`. The `progress` command is an additional command only available in the backend which reports the progress of a long-running migration (which is executed in a thread).
+While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `stats`. The `stats` command reports the progress of a long-running migration (which is executed in a thread).
 
 The output of all commands except `stats` is the following (for a successful request):
 ```js
 enum MigrationState {
-    MIGRATION_RUNNING = "migration_running"
     MIGRATION_REQUIRED = "migration_required"
+    MIGRATION_RUNNING = "migration_running"
+    MIGRATION_FAILED = "migration_failed"
     FINALIZATION_REQUIRED = "finalization_required"
-    NO_MIGRATION_REQUIRED = "no_migration_required"
+    FINALIZATION_RUNNING = "finalization_running"
+    FINALIZED = "finalized"
+    FINALIZATION_FAILED = "finalization_failed"
 }
 
 {
@@ -30,20 +33,20 @@ enum MigrationState {
 ```
 `output` always contains the full output of the migration command up to this point. `exception` contains the thrown exception, if any, which can only be the case if the command is finished (meaning `status != "migration_running"`). After issuing a migration command, it is waited a short period of time for the thread to finish, so the status can be all of these things for any command (e.g. after calling `migrate`, the returned status can be either `MIGRATION_RUNNING` if the migrations did not finish directly or `FINALIZATION_REQUIRED` if the migration is already done).
 
-The output of migration commands is stored until a new migration command is issued, meaning repeated `progress` requests after a finished command will always return the same result with the full output.
-
 The `stats` return value is the following:
 ```js
 {
     "success": True,
     "stats": {
         "status": MigrationState,
+        "output": str, // Optional
+        "exception": str, // Optional
         "current_migration_index": int,
         "target_migration_index": int,
-        "positions": int,
-        "events": int,
-        "partially_migrated_positions": int,
-        "fully_migrated_positions": int
+        "migratable_models": {
+            "count": int,
+            "migrated": int
+        }
     }
 }
 ```

entrypoint.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 
+printf "enter entrypoint.sh"
+set -e
+
+printf "\nOpenslides DBMS:\n"
+printf "Export env variables for database.\n"
 source scripts/export_database_variables.sh
 
 if [ ! $ANONYMOUS_ONLY ]; then
   meta/dev/scripts/wait-for-database.sh
+  printf "DBMS is started.\n"
 fi
 
 printf "Create schema.\n"

openslides_backend/action/actions/meeting/import_.py

@@ -5,7 +5,7 @@
 from typing import Any
 
 from openslides_backend.action.actions.meeting.mixins import MeetingPermissionMixin
-from openslides_backend.migrations import get_backend_migration_index
+from openslides_backend.migrations.migration_helper import MigrationHelper
 from openslides_backend.models.base import model_registry
 from openslides_backend.models.checker import Checker, CheckException
 from openslides_backend.models.fields import (
@@ -753,7 +753,7 @@ def check_data_migration_index(self, instance: dict[str, Any]) -> None:
         Check for valid migration index.
         """
         start_migration_index = instance["meeting"].pop("_migration_index")
-        backend_migration_index = get_backend_migration_index()
+        backend_migration_index = MigrationHelper.get_backend_migration_index()
         if backend_migration_index < start_migration_index:
             raise ActionException(
                 f"Your data migration index '{start_migration_index}' is higher than the migration index of this backend '{backend_migration_index}'! Please, update your backend!"

openslides_backend/action/actions/organization/initial_import.py

@@ -1,6 +1,8 @@
 from collections.abc import Iterable
 from typing import Any
 
+from openslides_backend.migrations.migration_helper import MigrationHelper
+
 from ....i18n.translator import Translator
 from ....i18n.translator import translate as _
 from ....models.base import json_dict_to_non_json_data_types
@@ -142,10 +144,7 @@ def build_write_request(
     def create_action_result_element(
         self, instance: dict[str, Any]
     ) -> ActionResultElement | None:
-        backend_migration_index = 1
-        # TODO set to fixed value because of lacking migrations
-        # needs to be readded in some shape or form
-        # backend_migration_index = get_backend_migration_index()
+        backend_migration_index = MigrationHelper.get_backend_migration_index()
         result = {
             "data_migration_index": self.data_migration_index,
             "backend_migration_index": backend_migration_index,

openslides_backend/http/views/action_view.py

@@ -5,9 +5,10 @@
 from ...action.action_handler import ActionHandler
 from ...action.action_worker import handle_action_in_worker_thread
 from ...i18n.translator import Translator
-from ...migrations import assert_migration_index
-from ...migrations.migration_handler import MigrationHandler
+from ...migrations.migration_helper import MigrationHelper
+from ...migrations.migration_manager import MigrationManager
 from ...services.auth.interface import AUTHENTICATION_HEADER, COOKIE_NAME
+from ...services.postgresql.db_connection_handling import get_new_os_conn
 from ...shared.env import DEV_PASSWORD
 from ...shared.exceptions import AuthenticationException, ServerError
 from ...shared.interfaces.wsgi import RouteResponse
@@ -31,8 +32,9 @@ class ActionView(BaseView):
     def action_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching action request.")
 
-        assert_migration_index()
-
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
         # Get user id.
         user_id, access_token = self.get_user_id_from_headers(
             request.headers, request.cookies
@@ -56,7 +58,9 @@ def action_route(self, request: Request) -> RouteResponse:
     def internal_action_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching internal action request.")
 
-        assert_migration_index()
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
         self.check_internal_auth_password(request)
 
         handler = ActionHandler(self.env, self.services, self.logging)
@@ -71,8 +75,8 @@ def internal_action_route(self, request: Request) -> RouteResponse:
     def migrations_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start executing migrations request.")
         self.check_internal_auth_password(request)
-        handler = MigrationHandler(self.env, self.services, self.logging)
-        response = handler.handle_request(request.json)
+        manager = MigrationManager(self.env, self.services, self.logging)
+        response = manager.handle_request(request.json)
         self.logger.debug("Migrations request finished successfully.")
         return {"success": True, **response}, None
 

openslides_backend/http/views/presenter_view.py

@@ -1,5 +1,6 @@
-from ...migrations import assert_migration_index
+from ...migrations.migration_helper import MigrationHelper
 from ...presenter.presenter import PresenterHandler
+from ...services.postgresql.db_connection_handling import get_new_os_conn
 from ...shared.interfaces.wsgi import RouteResponse
 from ..request import Request
 from .base_view import BaseView, route
@@ -17,7 +18,9 @@ class PresenterView(BaseView):
     def presenter_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching presenter request.")
 
-        assert_migration_index()
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
 
         # Handle request.
         handler = PresenterHandler(

openslides_backend/migrations/README.md

@@ -1,32 +1,21 @@
-# Migrations for the Datastore
+# TODO revise this file
+# Migrations for the Database
 
-The `migrate.py` script is the main entrypoint to execute migrations. The migrations itself are in the `migrations` folder. Each file can be arbitrarily named but must include a class called `Migration` that can be found by the loader. Note that all docker related files in this folder are only for **developing** migrations, not execute them. For execution in the dev mode, see the actual backend target for the dockerfile (`target dev`) and `dev/entrypoint.sh`.
+The migrations will create shadow copies with `{name}_mig` of all tables to be migrated and rereference all triggers and references to those. Migration will be done on the `_mig`-tables. In the finalization step the original tables are deleted and the shadow copies put into place.
+Collections in READ_MODELS will prevent writing on corresponding tables for all parrallel processes.
+Collections in WRITE_MODELS will additionally prevent reading on corresponding tables for all parrallel processes.
 
-There are three ways to execute migrations:
-
-## 1) Developing migrations
-This is done within the dockersetup in this folder. Note that the datastore needs to be checked out with the name `openslides-datastore-service` next to the backend repository.
-
-- checkout the current datastore (in ../../openslides-datastore-service) and the current backend.
-- `make dev`: Starts the compose setup -> A shell opens with the ability to run `migrate.py` and auxillary scripts.
-- You can exit from it with `exit` and shut down the docker setup with `make stop-dev`
-- You can write migrations in the backend and also adjust the Datastore at the same time since both are mounted into the container.
-
-The following scripts can be used to make snapshots and trying out new migrations
+The migrations themselves are in the `migrations` folder. Each file must start with a four digit number and can include certain functions that can be found by the loader. A psycopg cursor object will be passed as function parameter.
+ * data_definition: should do necessary schema changes
+ * data_manipulation: should pass the altered original data from the table to table_mig
+ * cleanup: should do all cleanups that aren't done automatically like deleting additional temporary tables. This step happens during finalization before all automatic changes.
 
 #### Scripts for setting initial data
-Only the current dataset is exported, so after a (re-)import, only one position exists in the datastore. Also note that importing clears the old content
 
-- `export-data-only.sh` [to:export.json]
-- `import-data-only.sh` [from:export.json]
-
-#### Scripts for the full backup (Does a DB dump)
-- `export-events.sh` [to:export.sql]
-- `import-events.sh` [from:export.sql]
-
-#### Downloading example data
-- `fetch-example-data.sh` [to:example-data.json]
+There are three ways to execute migrations:
 
+## 1) The `migrate.py` script is the cli entrypoint to execute migrations.
+Use python `migrate.py -h` to see all available commands.
 
 ## 2) Migrations in dev mode