⚗️ msgspec + nutree + FormioConfig rewrite

POC for a proper data types rewrite to deal with anything formio in the backend.

See #6565

Author: sergei-maertens

bin/report_component_problems.py

@@ -193,6 +193,57 @@ def check_component(component: Component) -> Iterator[str]:
                             f"{language_code}."
                         )
 
+        case {"type": "editgrid"}:
+            if "groupLabel" not in component:
+                yield "Missing 'groupLabel' property."
+
+    # detect inconsistent default values
+    match component:
+        case {
+            "type": "textfield"
+            | "email"
+            | "phoneNumber"
+            | "postcode"
+            | "textarea"
+            | "select"
+            | "date"
+            | "datetime"
+            | "time",
+            "multiple": True,
+            "defaultValue": str(),
+        }:
+            yield "non-array defaultValue for 'multiple: true'"
+
+        case {
+            "type": "textfield"
+            | "email"
+            | "phoneNumber"
+            | "postcode"
+            | "textarea"
+            | "select"
+            | "date"
+            | "datetime"
+            | "time",
+            "multiple": True,
+            "defaultValue": list() as dv,
+        }:
+            if None in dv:
+                yield "None found in defaultValue for text-based component"
+
+        case {
+            "type": "textfield"
+            | "email"
+            | "phoneNumber"
+            | "postcode"
+            | "textarea"
+            | "select"
+            | "date"
+            | "datetime"
+            | "time",
+            "defaultValue": list(),
+        } if not (multiple := component.get("multiple")):
+            yield f"array defaultValue for 'multiple: {multiple}'"
+
 
 def check_component_html_usage(component: Component) -> list[str]:
     messages = []

bin/report_conditional_eq_properties.py

@@ -71,7 +71,7 @@ def report_configurations() -> bool:
     known_conditional_keys = {"when", "show"}
     for form_definition in form_definitions.iterator(chunk_size=10):
         component_keys: set[str] = set()
-        for component in form_definition.configuration_wrapper:
+        for component in form_definition.iter_components():
             if not (conditional := component.get("conditional")):
                 continue
             elif (

bin/report_logic_with_deprecated_clear_on_hide_behavior.py

@@ -10,11 +10,14 @@
 import django
 
 import click
+import msgspec
 from json_logic.meta import JSONLogicExpression
 from json_logic.meta.expressions import destructure
 from json_logic.typing import JSON
 from tabulate import tabulate
 
+from formio_types import AnyComponent, EditGrid, FormioConfiguration, Selectboxes
+
 SRC_DIR = Path(__file__).parent.parent / "src"
 sys.path.insert(0, str(SRC_DIR.resolve()))
 
@@ -86,15 +89,11 @@ def analyze_rule(
     *,
     rule,
     form,
-    component_map,
+    component_map: dict[str, AnyComponent],
     components_with_affected_visibility,
     data,
 ):
-    from openforms.formio.service import (
-        get_component_empty_value,
-        iter_components,
-    )
-    from openforms.formio.typing import Component
+    from openforms.formio.service import get_component_empty_value, iter_components
     from openforms.forms.constants import LogicActionTypes
     from openforms.variables.service import resolve_key
 
@@ -110,14 +109,14 @@ def analyze_rule(
             continue
 
         component = component_map[resolved_key]
-        if component["type"] == "editgrid" and var_name != resolved_key:
+        if isinstance(component, EditGrid) and var_name != resolved_key:
             # We expect data access "editgrid.x.child_key" here, so discard the
             # parent key and index. Assuming there are no nested editgrids here
             # :see_no_evil:
             _, child_key = var_name.removeprefix(f"{resolved_key}.").split(".", 1)
 
-            children_map: dict[str, Component] = {
-                child["key"]: child
+            children_map: dict[str, AnyComponent] = {
+                child.key: child
                 for child in iter_components(
                     component, recursive=True, recurse_into_editgrid=False
                 )
@@ -128,13 +127,13 @@ def analyze_rule(
 
         # Visibility of component is not affected and/or component does not have
         # clearOnHide enabled, so it's not relevant
-        if resolved_key not in components_with_affected_visibility or not component.get(
-            "clearOnHide", True
+        if resolved_key not in components_with_affected_visibility or not getattr(
+            component, "clear_on_hide", True
         ):
             continue
 
         empty_value = get_component_empty_value(component)
-        if component["type"] == "selectboxes":
+        if isinstance(component, Selectboxes):
             # `get_component_empty_value` returns {"option_a": False, "option_b": False, etc...}
             # for a selectboxes component, which is not a useful in this
             # context. It is not possible to use a dictionary as a comparison
@@ -148,7 +147,7 @@ def analyze_rule(
         # current default that is set when a variable is missing from the
         # context. Note that all form variables should be present in the context
         # at the moment, but there is no such guarantee for nested data.
-        if comp_value in [empty_value, None, component.get("defaultValue")]:
+        if comp_value in [empty_value, None, getattr(component, "default_value", None)]:
             variable_names.add(var_name)
 
     if variable_names:
@@ -218,8 +217,8 @@ def analyze_rule(
 
 
 def report_rules() -> bool:
-    from openforms.formio.service import iter_components
-    from openforms.formio.typing import Component
+
+    from openforms.formio.service import _fixup_component_properties, iter_components
     from openforms.formio.visibility import get_conditional
     from openforms.forms.models import Form
 
@@ -233,18 +232,23 @@ def report_rules() -> bool:
 
         # Mapping from component to step for quick access
         form_steps = form.formstep_set.select_related("form_definition")
-        component_map: dict[str, Component] = {}
+        component_map: dict[str, AnyComponent] = {}
         for form_step in form_steps:
-            for component in iter_components(
+            formio_configuration = msgspec.convert(
                 form_step.form_definition.configuration,
+                type=FormioConfiguration,
+                dec_hook=_fixup_component_properties,
+            )
+            for component in iter_components(
+                formio_configuration,
                 recursive=True,
                 recurse_into_editgrid=False,
             ):
-                component_map[component["key"]] = component
+                component_map[component.key] = component
 
                 # Component with visibility affected by a conditional
                 if get_conditional(component) is not None:
-                    components_with_affected_visibility.add(component["key"])
+                    components_with_affected_visibility.add(component.key)
 
         # Components with visibility affected by logic rules
         for rule in form.formlogic_set.iterator():
@@ -261,7 +265,7 @@ def report_rules() -> bool:
 
             component = component_map[key]
             children = {
-                child["key"]
+                child.key
                 for child in iter_components(
                     component, recursive=True, recurse_into_editgrid=False
                 )

pyright.pyproject.toml

@@ -30,12 +30,22 @@ include = [
     "src/openforms/config/forms.py",
     "src/openforms/config/api/",
     # Formio tooling
+    "src/openforms/formio/components/translations.py",
+    "src/openforms/formio/dynamic_config/date.py",
+    "src/openforms/formio/dynamic_config/reference_lists.py",
+    "src/openforms/formio/dynamic_config/tests/test_reference_lists_config.py",
+    "src/openforms/formio/formatters/",
+    "src/openforms/formio/rendering/default.py",
+    "src/openforms/formio/rendering/nodes.py",
+    "src/openforms/formio/rendering/tests/test_component_node.py",
+    "src/openforms/formio/rendering/tests/test_custom_formio_components.py",
+    "src/openforms/formio/rendering/tests/test_vanilla_formio_components.py",
     "src/openforms/formio/serializers.py",
     "src/openforms/formio/service.py",
-    "src/openforms/formio/typing/",
-    "src/openforms/formio/formatters/",
-    "src/openforms/formio/dynamic_config/reference_lists.py",
     "src/openforms/formio/tests/test_datastructures.py",
+    "src/openforms/formio/tests/test_visibility.py",
+    "src/openforms/formio/typing/",
+    # "src/openforms/formio/tests/test_component_translations.py",
     # Core forms app
     "src/openforms/forms/api/serializers/logic/action_serializers.py",
     "src/openforms/forms/api/v3/",
@@ -73,6 +83,8 @@ include = [
     "src/openforms/registrations/contrib/stuf_zds/plugin.py",
     "src/openforms/registrations/contrib/stuf_zds/typing.py",
     "src/openforms/registrations/contrib/zgw_apis/",
+    "src/openforms/registrations/tests/test_component_pre_registration_tasks.py",
+    "src/openforms/registrations/tests/test_process_variable_schema.py",
     # Translations
     "src/openforms/translations/api/views.py",
     "src/openforms/translations/admin.py",
@@ -92,6 +104,7 @@ include = [
     "src/openforms/submissions/logic/actions.py",
     "src/openforms/submissions/metrics.py",
     "src/openforms/submissions/query.py",
+    "src/openforms/submissions/rendering/nodes.py",
     "src/openforms/submissions/report.py",
     "src/openforms/submissions/tests/form_logic/test_get_rules_to_evaluate.py",
     "src/openforms/submissions/tests/form_logic/test_rule_analysis.py",
@@ -117,7 +130,7 @@ exclude = [
     "src/openforms/authentication/contrib/eherkenning/tests/test_signicat_integration.py",
     "src/openforms/contrib/objects_api/tests/",
     "src/openforms/contrib/objects_api/json_schema.py",
-    "src/openforms/formio/formatters/tests/",
+    "src/openforms/formio/formatters/tests/test_default_formatters.py",
     "src/openforms/payments/management/commands/checkpaymentemaildupes.py",
     "src/openforms/payments/tests/",
     "src/openforms/payments/contrib/demo/tests/",

requirements/base.in

@@ -13,7 +13,9 @@ jsonschema_specifications
 jq
 lxml
 lxml-html-clean
+msgspec
 nh3
+nutree
 onlinepayments-sdk-python3 # Worldline SDK
 O365  # microsoft graph
 phonenumberslite

requirements/base.txt

@@ -365,12 +365,16 @@ mozilla-django-oidc-db==1.1.1
     #   django-digid-eherkenning
 msal==1.37.0
     # via o365
+msgspec==0.21.0
+    # via -r requirements/base.in
 networkx==3.6.1
     # via -r requirements/base.in
 nh3==0.3.3
     # via -r requirements/base.in
 numpy==2.4.2
     # via shapely
+nutree==1.1.0
+    # via -r requirements/base.in
 o365==2.1.9
     # via
     #   -r requirements/base.in
@@ -648,6 +652,7 @@ typing-extensions==4.15.0
     #   django-timeline-logger
     #   grpcio
     #   mozilla-django-oidc-db
+    #   nutree
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http

requirements/ci.txt

@@ -633,6 +633,10 @@ msal==1.37.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   o365
+msgspec==0.21.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 networkx==3.6.1
     # via
     #   -c requirements/base.txt
@@ -646,6 +650,10 @@ numpy==2.4.2
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   shapely
+nutree==1.1.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 o365==2.1.9
     # via
     #   -c requirements/base.txt
@@ -1161,6 +1169,7 @@ typing-extensions==4.15.0
     #   django-timeline-logger
     #   grpcio
     #   mozilla-django-oidc-db
+    #   nutree
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http

requirements/dev.txt

@@ -694,6 +694,10 @@ msal==1.37.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   o365
+msgspec==0.21.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 networkx==3.6.1
     # via
     #   -c requirements/ci.txt
@@ -707,6 +711,10 @@ numpy==2.4.2
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   shapely
+nutree==1.1.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 o365==2.1.9
     # via
     #   -c requirements/ci.txt
@@ -1294,6 +1302,7 @@ typing-extensions==4.15.0
     #   django-timeline-logger
     #   grpcio
     #   mozilla-django-oidc-db
+    #   nutree
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http

requirements/extensions.txt

@@ -591,6 +591,10 @@ msal==1.37.0
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   o365
+msgspec==0.21.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 networkx==3.6.1
     # via
     #   -c requirements/base.txt
@@ -604,6 +608,10 @@ numpy==2.4.2
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   shapely
+nutree==1.1.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 o365==2.1.9
     # via
     #   -c requirements/base.txt
@@ -1063,6 +1071,7 @@ typing-extensions==4.15.0
     #   django-timeline-logger
     #   grpcio
     #   mozilla-django-oidc-db
+    #   nutree
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http

requirements/type-checking.txt

@@ -680,6 +680,10 @@ msal==1.37.0
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   o365
+msgspec==0.21.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 networkx==3.6.1
     # via
     #   -c requirements/ci.txt
@@ -693,6 +697,10 @@ numpy==2.4.2
     #   -c requirements/ci.txt
     #   -r requirements/ci.txt
     #   shapely
+nutree==1.1.0
+    # via
+    #   -c requirements/ci.txt
+    #   -r requirements/ci.txt
 o365==2.1.9
     # via
     #   -c requirements/ci.txt
@@ -1281,6 +1289,7 @@ typing-extensions==4.15.0
     #   djangorestframework-stubs
     #   grpcio
     #   mozilla-django-oidc-db
+    #   nutree
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http