Refactor auth (#891)

* Refactor API key handling

* Clarify registry usage

* Create routes after registries known

* FIx minor issues from refactors

* Nit: remove Security scopes when not used

* Fix CI

* Refactor Scope checking

* refactor SecureEntry into standard Security

* Remove unused dependency_overrides

* Remove unused parameters

* Document requirement for APIKeyHeader handling

* Clarify returned Scopes

* Prevent mutation of Scopes

* Prevent closure of QueryRegistry

* Prevent trying to edit frozenset

* Add changelog entries

* Resolve missed merge conflicts

* Amend changelog

Author: DiamondJoseph

CHANGELOG.md

@@ -9,6 +9,12 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 - `Composite` structure family to enable direct access to table columns in a single namespace.
 
+### Maintenance
+
+- Extract API key handling
+- Extract scope fetching and checking
+- Refactor router construction
+
 
 ## 0.1.0-b20 (2025-03-07)
 

docs/source/explanations/access-control.md

@@ -70,6 +70,7 @@ integrate with our proposal system.
 import cachetools
 import httpx
 from tiled.queries import In
+from tiled.scopes import PUBLIC_SCOPES
 
 
 # To reduce load on the external service and to expedite repeated lookups, use a
@@ -102,12 +103,12 @@ class PASSAccessPolicy:
             )
 
     def allowed_scopes(self, node, principal, path_parts):
-        return {"read:metadata", "read:data"}
+        return PUBLIC_SCOPES
 
     def filters(self, node, principal, scopes, path_parts):
         queries = []
         id = self._get_id(principal)
-        if not scopes.issubset({"read:metadata", "read:data"}):
+        if not scopes.issubset(PUBLIC_SCOPES):
             return NO_ACCESS
         try:
             response = response_cache[id]

docs/source/reference/service.md

@@ -79,7 +79,6 @@ When registering new types, make reference to the
 .. autosummary::
    :toctree: generated
 
-   tiled.media_type_registration.serialization_registry
    tiled.media_type_registration.SerializationRegistry
    tiled.media_type_registration.SerializationRegistry.register
    tiled.media_type_registration.SerializationRegistry.media_types

tiled/_tests/test_access_control.py

@@ -5,16 +5,12 @@
 from fastapi import HTTPException
 from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
 
-from ..access_policies import (
-    ALL_SCOPES,
-    PUBLIC_SCOPES,
-    SimpleAccessPolicy,
-    SpecialUsers,
-)
+from ..access_policies import SimpleAccessPolicy, SpecialUsers
 from ..adapters.array import ArrayAdapter
 from ..adapters.mapping import MapAdapter
 from ..client import Context, from_context
 from ..client.utils import ClientError
+from ..scopes import ALL_SCOPES, PUBLIC_SCOPES
 from ..server.app import build_app_from_config
 from ..server.core import NoEntry
 from .utils import enter_username_password, fail_with_status_code
@@ -36,8 +32,6 @@ async def allowed_scopes(self, node, principal, path_parts):
         # If this is being called, filter_access has let us get this far.
         if principal is SpecialUsers.public:
             allowed = PUBLIC_SCOPES
-        elif principal.type == "service":
-            allowed = self.scopes
         else:
             allowed = self.scopes
 
@@ -64,7 +58,7 @@ async def allowed_scopes(self, node, principal, path_parts):
                 )
             remove_scope = node.metadata().get("remove_scope", None)
             if remove_scope in allowed:
-                allowed = allowed.copy()
+                allowed = set(allowed)
                 allowed.remove(remove_scope)
         return allowed
 

tiled/_tests/test_protocols.py

@@ -9,7 +9,7 @@
 from numpy.typing import NDArray
 from pytest_mock import MockFixture
 
-from tiled.access_policies import ALL_ACCESS, ALL_SCOPES
+from tiled.access_policies import ALL_ACCESS
 from tiled.adapters.awkward_directory_container import DirectoryContainer
 from tiled.adapters.protocols import (
     AccessPolicy,
@@ -19,6 +19,7 @@
     SparseAdapter,
     TableAdapter,
 )
+from tiled.scopes import ALL_SCOPES
 from tiled.server.schemas import Principal, PrincipalType
 from tiled.structures.array import ArrayStructure, BuiltinDtype
 from tiled.structures.awkward import AwkwardStructure

tiled/access_policies.py

@@ -1,12 +1,10 @@
 from functools import partial
 
 from .queries import In, KeysFilter
-from .scopes import SCOPES
+from .scopes import ALL_SCOPES, PUBLIC_SCOPES
 from .utils import Sentinel, SpecialUsers, import_object
 
 ALL_ACCESS = Sentinel("ALL_ACCESS")
-ALL_SCOPES = set(SCOPES)
-PUBLIC_SCOPES = {"read:metadata", "read:data"}
 NO_ACCESS = Sentinel("NO_ACCESS")
 
 

tiled/client/container.py

@@ -15,7 +15,7 @@
 from ..adapters.utils import IndexersMixin
 from ..iterviews import ItemsView, KeysView, ValuesView
 from ..queries import KeyLookup
-from ..query_registration import query_registry
+from ..query_registration import default_query_registry
 from ..structures.core import Spec, StructureFamily
 from ..structures.data_source import DataSource
 from ..utils import UNCHANGED, OneShotCachedMap, Sentinel, node_repr, safe_json_dump
@@ -1205,7 +1205,7 @@ def _queries_to_params(*queries):
     "Compute GET params from the queries."
     params = collections.defaultdict(list)
     for query in queries:
-        name = query_registry.query_type_to_name[type(query)]
+        name = default_query_registry.query_type_to_name[type(query)]
         for field, value in query.encode().items():
             if value is not None:
                 params[f"filter[{name}][condition][{field}]"].append(value)

tiled/config.py

@@ -10,19 +10,21 @@
 from datetime import timedelta
 from functools import cache
 from pathlib import Path
+from typing import Optional
 
 import jsonschema
 
 from .adapters.mapping import MapAdapter
 from .media_type_registration import (
-    compression_registry as default_compression_registry,
+    CompressionRegistry,
+    SerializationRegistry,
+    default_compression_registry,
+    default_deserialization_registry,
+    default_serialization_registry,
 )
-from .media_type_registration import (
-    serialization_registry as default_serialization_registry,
-)
-from .query_registration import query_registry as default_query_registry
+from .query_registration import QueryRegistry, default_query_registry
 from .utils import import_object, parse, prepend_to_sys_path
-from .validation_registration import validation_registry as default_validation_registry
+from .validation_registration import ValidationRegistry, default_validation_registry
 
 
 @cache
@@ -40,10 +42,11 @@ def construct_build_app_kwargs(
     config,
     *,
     source_filepath=None,
-    query_registry=None,
-    compression_registry=None,
-    serialization_registry=None,
-    validation_registry=None,
+    query_registry: Optional[QueryRegistry] = None,
+    compression_registry: Optional[CompressionRegistry] = None,
+    serialization_registry: Optional[SerializationRegistry] = None,
+    deserialization_registry: Optional[SerializationRegistry] = None,
+    validation_registry: Optional[ValidationRegistry] = None,
 ):
     """
     Given parsed configuration, construct arguments for build_app(...).
@@ -61,6 +64,8 @@ def construct_build_app_kwargs(
         query_registry = default_query_registry
     if serialization_registry is None:
         serialization_registry = default_serialization_registry
+    if deserialization_registry is None:
+        deserialization_registry = default_deserialization_registry
     if compression_registry is None:
         compression_registry = default_compression_registry
     if validation_registry is None:
@@ -220,6 +225,7 @@ def construct_build_app_kwargs(
         "server_settings": server_settings,
         "query_registry": query_registry,
         "serialization_registry": serialization_registry,
+        "deserialization_registry": deserialization_registry,
         "compression_registry": compression_registry,
         "validation_registry": validation_registry,
         "tasks": {

tiled/media_type_registration.py

@@ -197,21 +197,21 @@ def __call__(self, media_type, encoder, *args, **kwargs):
         return self.dispatch(media_type, encoder)(*args, **kwargs)
 
 
-serialization_registry = SerializationRegistry()
+default_serialization_registry = SerializationRegistry()
 "Global serialization registry. See Registry for usage examples."
 
-deserialization_registry = SerializationRegistry()
+default_deserialization_registry = SerializationRegistry()
 "Global deserialization registry. See Registry for usage examples."
 
-compression_registry = CompressionRegistry()
+default_compression_registry = CompressionRegistry()
 "Global compression registry. See Registry for usage examples."
 
 
 for media_type in [
     "application/json",
     "application/x-msgpack",
 ]:
-    compression_registry.register(
+    default_compression_registry.register(
         media_type,
         "gzip",
         lambda buffer: gzip.GzipFile(mode="wb", fileobj=buffer, compresslevel=9),
@@ -225,7 +225,7 @@ def __call__(self, media_type, encoder, *args, **kwargs):
     "text/plain",
     "text/html",
 ]:
-    compression_registry.register(
+    default_compression_registry.register(
         media_type,
         "gzip",
         # Use a lower compression level. High compression is extremely slow
@@ -270,7 +270,7 @@ def close(self):
         "text/html",
         "text/plain",
     ]:
-        compression_registry.register(media_type, "zstd", ZstdBuffer)
+        default_compression_registry.register(media_type, "zstd", ZstdBuffer)
 
 if modules_available("lz4"):
     import lz4
@@ -326,7 +326,7 @@ def close(self):
         "text/html",
         "text/plain",
     ]:
-        compression_registry.register(media_type, "lz4", LZ4Buffer)
+        default_compression_registry.register(media_type, "lz4", LZ4Buffer)
 
 if modules_available("blosc2"):
     import blosc2
@@ -355,4 +355,4 @@ def close(self):
             pass
 
     for media_type in ["application/octet-stream", APACHE_ARROW_FILE_MIME_TYPE]:
-        compression_registry.register(media_type, "blosc2", BloscBuffer)
+        default_compression_registry.register(media_type, "blosc2", BloscBuffer)

tiled/query_registration.py

@@ -79,8 +79,8 @@ def inner(cls):
 
 
 # Make a global registry.
-query_registry = QueryRegistry()
-register = query_registry.register
+default_query_registry = QueryRegistry()
+register = default_query_registry.register
 """Register a new type of query."""
 
 

tiled/scopes.py

@@ -19,3 +19,18 @@
         "description": "Edit list of all users and services and their attributes."
     },
 }
+
+ALL_SCOPES: set[str] = frozenset(SCOPES)
+PUBLIC_SCOPES: set[str] = frozenset(("read:metadata", "read:data"))
+USER_SCOPES: set[str] = frozenset(
+    (
+        "read:metadata",
+        "read:data",
+        "write:metadata",
+        "write:data",
+        "create",
+        "register",
+        "metrics",
+    )
+)
+NO_SCOPES: set[str] = frozenset()

tiled/serialization/array.py

@@ -3,7 +3,10 @@
 
 import numpy
 
-from ..media_type_registration import deserialization_registry, serialization_registry
+from ..media_type_registration import (
+    default_deserialization_registry,
+    default_serialization_registry,
+)
 from ..utils import (
     SerializationError,
     UnsupportedShape,
@@ -22,13 +25,13 @@ def as_buffer(array, metadata):
         return numpy.asarray(array).tobytes()
 
 
-serialization_registry.register(
+default_serialization_registry.register(
     "array",
     "application/octet-stream",
     as_buffer,
 )
 if modules_available("orjson"):
-    serialization_registry.register(
+    default_serialization_registry.register(
         "array",
         "application/json",
         lambda array, metadata: safe_json_dump(array),
@@ -43,10 +46,12 @@ def serialize_csv(array, metadata):
     return file.getvalue().encode()
 
 
-serialization_registry.register("array", "text/csv", serialize_csv)
-serialization_registry.register("array", "text/x-comma-separated-values", serialize_csv)
-serialization_registry.register("array", "text/plain", serialize_csv)
-deserialization_registry.register(
+default_serialization_registry.register("array", "text/csv", serialize_csv)
+default_serialization_registry.register(
+    "array", "text/x-comma-separated-values", serialize_csv
+)
+default_serialization_registry.register("array", "text/plain", serialize_csv)
+default_deserialization_registry.register(
     "array",
     "application/octet-stream",
     lambda buffer, dtype, shape: numpy.frombuffer(buffer, dtype=dtype).reshape(shape),
@@ -90,10 +95,10 @@ def array_from_buffer_PIL(buffer, format, dtype, shape):
         image = Image.open(file, format=format)
         return numpy.asarray(image).asdtype(dtype).reshape(shape)
 
-    serialization_registry.register(
+    default_serialization_registry.register(
         "array", "image/png", lambda array, metadata: save_to_buffer_PIL(array, "png")
     )
-    deserialization_registry.register(
+    default_deserialization_registry.register(
         "array",
         "image/png",
         lambda buffer, dtype, shape: array_from_buffer_PIL(buffer, "png", dtype, shape),
@@ -120,18 +125,24 @@ def save_to_buffer_tifffile(array, metadata):
         imwrite(file, normalized_array)
         return file.getbuffer()
 
-    serialization_registry.register("array", "image/tiff", save_to_buffer_tifffile)
-    deserialization_registry.register("array", "image/tiff", array_from_buffer_tifffile)
+    default_serialization_registry.register(
+        "array", "image/tiff", save_to_buffer_tifffile
+    )
+    default_deserialization_registry.register(
+        "array", "image/tiff", array_from_buffer_tifffile
+    )
 
 
 def serialize_html(array, metadata):
     "Try to display as image. Fall back to CSV."
     try:
-        png_data = serialization_registry.dispatch("array", "image/png")(
+        png_data = default_serialization_registry.dispatch("array", "image/png")(
             array, metadata
         )
     except Exception:
-        csv_data = serialization_registry.dispatch("array", "text/csv")(array, metadata)
+        csv_data = default_serialization_registry.dispatch("array", "text/csv")(
+            array, metadata
+        )
         return "<html>" "<body>" f"{csv_data.decode()!s}" "</body>" "</html>"
     else:
         return (
@@ -145,4 +156,4 @@ def serialize_html(array, metadata):
         )
 
 
-serialization_registry.register("array", "text/html", serialize_html)
+default_serialization_registry.register("array", "text/html", serialize_html)