Refactor model discovery and introduce tags.

Author: RoelBollens-TomTom

README.pydantic.md

@@ -149,26 +149,26 @@ Registration is done in the `[project.entry-points."overture.models"]` section:
 
 ```toml
 [project.entry-points."overture.models"]
-"buildings.building" = "overture.schema.buildings.building.models:Building"
-"buildings.building_part" = "overture.schema.buildings.building_part.models:BuildingPart"
+building = "overture.schema.buildings.building.models:Building"
+building_part = "overture.schema.buildings.building_part.models:BuildingPart"
 ```
 
 The discovery system provides programmatic access to registered models:
 
 ```python
-from overture.schema.core.discovery import discover_models, get_registered_model
+from overture.schema.system.discovery import discover_models, get_registered_model
 
 # Discover all registered models
 all_models = discover_models()
 # Returns:
 # {
-#   ("buildings", "building"): BuildingModel,
-#   ("places", "place"): PlaceModel,
+#   ("building", "acme:Building", {"building_tag"}): BuildingModel,
+#   ("place", "acme:Place", {"place_tag"}): PlaceModel, 
 # ...
 # }
 
-# Get a specific model by theme and type
-building_model = get_registered_model("buildings", "building")
+# Get a specific model by type
+building_model = get_registered_model("building")
 if building_model:
     # Use the model class
     building = building_model.model_validate(building_data)

packages/overture-schema-addresses-theme/pyproject.toml

@@ -37,4 +37,4 @@ pythonpath = ["src"]
 testpaths = ["tests"]
 
 [project.entry-points."overture.models"]
-"overture:addresses:address" = "overture.schema.addresses:Address"
+address = "overture.schema.addresses:Address"

packages/overture-schema-annex/pyproject.toml

@@ -29,7 +29,7 @@ path = "src/overture/schema/__about__.py"
 packages = ["src/overture"]
 
 [project.entry-points."overture.models"]
-"annex:sources" = "overture.schema.annex:Sources"
+sources = "overture.schema.annex:Sources"
 
 [tool.pytest.ini_options]
 pythonpath = ["src"]

packages/overture-schema-base-theme/pyproject.toml

@@ -35,9 +35,9 @@ path = "src/overture/schema/base/__about__.py"
 packages = ["src/overture"]
 
 [project.entry-points."overture.models"]
-"overture:base:bathymetry" = "overture.schema.base:Bathymetry"
-"overture:base:infrastructure" = "overture.schema.base:Infrastructure"
-"overture:base:land" = "overture.schema.base:Land"
-"overture:base:land_cover" = "overture.schema.base:LandCover"
-"overture:base:land_use" = "overture.schema.base:LandUse"
-"overture:base:water" = "overture.schema.base:Water"
+bathymetry = "overture.schema.base:Bathymetry"
+infrastructure = "overture.schema.base:Infrastructure"
+land = "overture.schema.base:Land"
+land_cover = "overture.schema.base:LandCover"
+land_use = "overture.schema.base:LandUse"
+water = "overture.schema.base:Water"

packages/overture-schema-buildings-theme/pyproject.toml

@@ -35,5 +35,5 @@ path = "src/overture/schema/buildings/__about__.py"
 packages = ["src/overture"]
 
 [project.entry-points."overture.models"]
-"overture:buildings:building" = "overture.schema.buildings:Building"
-"overture:buildings:building_part" = "overture.schema.buildings:BuildingPart"
+building = "overture.schema.buildings:Building"
+building_part = "overture.schema.buildings:BuildingPart"

packages/overture-schema-cli/src/overture/schema/cli/commands.py

@@ -13,21 +13,20 @@
 import yaml
 from pydantic import BaseModel, Field, Tag, TypeAdapter, ValidationError
 from rich.console import Console
+from rich.text import Text
 from yamlcore import CoreLoader  # type: ignore
 
 from overture.schema.core import OvertureFeature
-from overture.schema.core.discovery import ModelKey, discover_models
+from overture.schema.system.discovery import ModelKey, discover_models, tags_by_key
 from overture.schema.system.feature import Feature
 from overture.schema.system.json_schema import json_schema
 
-from .docstrings import get_model_docstring, get_theme_module_docstring
 from .error_formatting import (
     format_validation_error,
     format_validation_errors_verbose,
     group_errors_by_discriminator,
     select_most_likely_errors,
 )
-from .output import rewrap
 from .type_analysis import StructuralTuple, get_item_index, introspect_union
 from .types import ErrorLocation, ModelDict, UnionType
 
@@ -207,34 +206,40 @@ def resolve_types(
     -------
         Model type suitable for passing to parse_feature
     """
-    # Determine effective namespace
-    effective_namespace = "overture" if use_overture_types else namespace
-
     # Discover models once with the appropriate namespace
-    all_models = discover_models(namespace=effective_namespace)
+    all_models = discover_models()
 
     # Filter models based on CLI options
     filtered_models: ModelDict = {}
 
+    if namespace and namespace != "overture":
+        filtered_models = {
+            key: model_class
+            for key, model_class in all_models.items()
+            if namespace in key.tags
+        }
+
     if use_overture_types:
-        filtered_models = all_models
+        for key, model_class in all_models.items():
+            if tags_by_key(key.tags, "overture:theme"):
+                filtered_models[key] = model_class
 
     elif theme_names and not type_names:
         # Theme-only mode: all types in specified themes
         for key, model_class in all_models.items():
-            if key.theme in theme_names:
+            if next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names:
                 filtered_models[key] = model_class
 
     elif type_names and not theme_names:
         # Type-only mode: find matching types across all themes
         for key, model_class in all_models.items():
-            if key.type in type_names:
+            if key.name in type_names and tags_by_key(key.tags, "overture:theme"):
                 filtered_models[key] = model_class
 
     elif type_names and theme_names:
         # Both specified: find matching types within specified themes
         for key, model_class in all_models.items():
-            if key.theme in theme_names and key.type in type_names:
+            if key.name in type_names and next(iter(tags_by_key(key.tags, "overture:theme")),None) in theme_names:
                 filtered_models[key] = model_class
 
     else:
@@ -767,49 +772,28 @@ def json_schema_command(
         raise click.UsageError(str(e)) from e
 
 
-def dump_namespace(
-    theme_types: dict[str | None, list[tuple[ModelKey, type[BaseModel]]]],
-) -> None:
-    """Print all themes and types for a namespace.
-
-    Displays themes in alphabetical order with their types and docstrings.
-    Each type includes its model class name and description.
-
-    Args
-    ----
-    theme_types : dict[str | None, list[tuple[ModelKey, type[BaseModel]]]]
-        Dict mapping theme name to list of (ModelKey, model_class) tuples
-    """
-    for theme in sorted(theme_types.keys(), key=lambda x: (x is None, x)):
-        if theme:
-            stdout.print(
-                f"[bold green underline]{theme.upper()}[/bold green underline]"
-            )
-
-            theme_docstring = get_theme_module_docstring(theme)
-            if theme_docstring:
-                stdout.print(
-                    rewrap(theme_docstring, stdout, padding_right=4), style="dim"
-                )
-
-            stdout.print()
-
-        # Add types to the tree
-        sorted_types = sorted(theme_types[theme], key=lambda x: x[0].type)
-        for key, model_class in sorted_types:
-            stdout.print(
-                f"  [bright_black]→[/bright_black] [bold cyan]{key.type}[/bold cyan] [dim magenta]({key.class_name})[/dim magenta]"
-            )
-            docstring = get_model_docstring(model_class)
-            if docstring:
-                stdout.print(
-                    rewrap(docstring, stdout, indent=4, padding_right=12), style="dim"
-                )
-            stdout.print()
-
-
 @cli.command("list-types")
-def list_types() -> None:
+@click.option(
+    "--tag",
+    "tags",
+    multiple=True,
+    help="Filter types by tag (e.g., overture:theme=addresses)",
+)
+@click.option(
+    "--exclude-tag",
+    "excluded_tags",
+    multiple=True,
+    help="Filter types by tag (e.g., overture:theme=base)",
+)
+@click.option(
+    "--group-by",
+    help="Group types by tag prefix (e.g., 'overture:theme')",
+)
+def list_types(
+    tags: tuple[str, ...],
+    excluded_tags: tuple[str, ...],
+    group_by: str | None
+) -> None:
     r"""List all available types grouped by theme with descriptions.
 
     Displays all registered Overture Maps types organized by theme,
@@ -822,35 +806,51 @@ def list_types() -> None:
     """
     try:
         models = discover_models()
+        filters = []
+
+        if tags:
+            filters.append(lambda key: all(tag in key.tags for tag in tags))
+        if excluded_tags:
+            filters.append(lambda key: not any(tag in key.tags for tag in excluded_tags))
+
+        if filters:
+            models = {
+                key: model
+                for key, model in models.items()
+                if all(f(key) for f in filters)
+            }
+
+        if group_by:
+            grouped_models: dict[str, set[ModelKey]] = {}
+
+            for key, model_class in models.items():
+                if groups := tags_by_key(key.tags, group_by):
+                    for group in groups:
+                        grouped_models.setdefault(group, set()).add(key)
+
+            padding = max((len(key.name) for keys in grouped_models.values() for key in keys), default=0) + 2
+
+            for group, keys in sorted(grouped_models.items()):
+                stdout.print(f"[green bold]{group_by}={group} ({len(keys)})[/green bold]")
+                for key in sorted(keys, key=lambda k: k.name):
+                    model = Text()
+                    model.append("→ ", style="bright_black")
+                    model.append(key.name, style="bold cyan")
+                    model.pad_right(max(1, padding - len(key.name)))
+                    model.append_text(Text().append("  ".join(sorted(key.tags))))
+                    stdout.print(model)
+                stdout.print()
+
+        else:
+            padding = max((len(key.name) for key in models.keys()), default=0) + 2
+
+            for key in sorted(models.keys(), key=lambda k: k.name):
+                model = Text()
+                model.append(key.name, style="bold cyan")
+                model.pad_right(max(1, padding - len(key.name)))
+                model.append_text(Text().append("  ".join(sorted(key.tags))))
+                stdout.print(model)
 
-        # Group models by namespace and theme
-        namespaces: dict[
-            str, dict[str | None, list[tuple[ModelKey, type[BaseModel]]]]
-        ] = {}
-        for key, model_class in models.items():
-            if key.namespace not in namespaces:
-                namespaces[key.namespace] = {}
-            if key.theme not in namespaces[key.namespace]:
-                namespaces[key.namespace][key.theme] = []
-
-            namespaces[key.namespace][key.theme].append((key, model_class))
-
-        # display Overture themes first
-        if "overture" in namespaces:
-            stdout.print("[bold red]OVERTURE THEMES[/bold red]", justify="center")
-            stdout.print()
-
-            dump_namespace(namespaces["overture"])
-
-            stdout.print("[bold red]ADDITIONAL TYPES[/bold red]", justify="center")
-            stdout.print()
-
-        for namespace in sorted(namespaces.keys()):
-            if namespace == "overture":
-                continue
-
-            stdout.print(f"[bold blue]{namespace.upper()}[/bold blue]")
-            dump_namespace(namespaces[namespace])
 
     except Exception as e:
         click.echo(f"Error listing types: {e}", err=True)

packages/overture-schema-cli/src/overture/schema/cli/types.py

@@ -5,7 +5,7 @@
 from pydantic import BaseModel
 from pydantic_core import ErrorDetails
 
-from overture.schema.core.discovery import ModelKey
+from overture.schema.system.discovery import ModelKey
 
 # Type alias for union types created from Pydantic models
 # This represents either a single model or a discriminated union of models

packages/overture-schema-cli/tests/test_resolve_types.py

@@ -2,6 +2,7 @@
 
 import pytest
 from overture.schema.cli.commands import resolve_types
+from overture.schema.system.discovery import tags_by_key
 
 
 class TestResolveTypes:
@@ -124,10 +125,10 @@ def test_resolve_types_returns_expected_themes(
         expected_themes: set[str],
     ) -> None:
         """Test that resolve_types returns models from expected themes."""
-        from overture.schema.core.discovery import discover_models
+        from overture.schema.system.discovery import discover_models
 
-        models = discover_models(namespace=namespace)
-        actual_themes = {key.theme for key in models.keys()}
+        models = discover_models()
+        actual_themes = {next(iter(tags_by_key(key.tags, "overture:theme")),None) for key in models.keys()}
 
         # Check that we have at least the expected themes (may have more)
         assert expected_themes.issubset(actual_themes), (

packages/overture-schema-core/pyproject.toml

@@ -36,3 +36,8 @@ dev = [
     "types-pyyaml>=6.0.12.20250516",
     "types-shapely>=2.1.0.20250710",
 ]
+
+[project.entry-points."overture.tag_providers"]
+overture  = "overture.schema.core.tag_providers:overture_provider"
+authority = "overture.schema.core.tag_providers:authority_provider"
+theme     = "overture.schema.core.tag_providers:theme_provider"