python: Add support for SCU-style template references.

VERSION

@@ -1 +1 @@
-8.1.0
+8.2.0

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "dazl"
-version = "8.1.0"
+version = "8.2.0"
 description = "high-level Ledger API client for Daml ledgers"
 license = "Apache-2.0"
 authors = ["Davin K. Tanabe <[email protected]>"]

python/dazl/__init__.py

@@ -57,4 +57,4 @@
     pass
 
 
-__version__ = "8.1.0"
+__version__ = "8.2.0"

python/dazl/damlast/lookup.py

@@ -14,6 +14,7 @@
 # potentially lead to performance degradations, particularly at application startup where type and
 # template lookups by name are very frequent. Please be conscious of the runtime costs of
 # modifications in this file!
+from collections import defaultdict
 import threading
 from types import MappingProxyType
 from typing import (
@@ -477,15 +478,16 @@ class MultiPackageLookup(SymbolLookup):
 
     def __init__(self, archives: Optional[Collection[Archive]] = None):
         self._lock = threading.Lock()
-        self._cache = dict[PackageRef, PackageLookup]()
+        self._cache_by_pkg_id = dict[PackageRef, PackageLookup]()
+        self._cache_by_name = dict[PackageRef, PackageLookup]()
         if archives is not None:
             self.add_archive(*archives)
 
     def archives(self) -> Collection[Archive]:
         """
         Return the list of known archives.
         """
-        return [lookup.archive for lookup in self._cache.values()]
+        return [lookup.archive for lookup in self._cache_by_pkg_id.values()]
 
     def add_archive(self, *a: Archive) -> None:
         """
@@ -496,27 +498,47 @@ def add_archive(self, *a: Archive) -> None:
 
         :param a: One or more :class:`Archive` instances to add.
         """
-        new_lookups = {ar.hash: PackageLookup(ar) for ar in a}
+        new_lookups_by_id = {ar.hash: PackageLookup(ar) for ar in a}
         with self._lock:
             # replace the old cache with a new one, incorporating values from both the existing
             # cache and the new lookups that came in. When there is a key conflict between the two
             # APIs, always prefer the existing cache to provide stability to callers.
-            self._cache = {**new_lookups, **self._cache}
-            LOG.debug("Updated package cache; now contains %d packages.", len(self._cache))
+            cache_by_id = {**new_lookups_by_id, **self._cache_by_pkg_id}
+
+            # (re)build the by-name package cache for smart contract upgrade (SCU) support;
+            # we also allow template lookups by package name, and will prefer the most recent
+            # version
+            lookups_by_name = defaultdict[PackageRef, list[PackageLookup]](list)
+            for package_lookup in cache_by_id.values():
+                if package_lookup.archive.package.metadata is not None:
+                    scu_name = PackageRef(f"#{package_lookup.archive.package.metadata.name}")
+                    lookups_by_name[scu_name].append(package_lookup)
+
+            cache_by_name = {
+                scu_pkg_ref: max(package_lookups, key=get_version_tuple)
+                for scu_pkg_ref, package_lookups in lookups_by_name.items()
+            }
+
+            self._cache_by_pkg_id = cache_by_id
+            self._cache_by_name = cache_by_name
+            LOG.debug(
+                "Updated package cache; now contains %d packages.", len(self._cache_by_pkg_id)
+            )
 
     def package_ids(self) -> AbstractSet[PackageRef]:
         """
         Return the package IDs of packages stored in this lookup.
 
         Because the data is local and in memory, the timeout parameter is ignored.
         """
-        return set(self._cache)
+        return set(self._cache_by_pkg_id)
 
     def package(self, ref: PackageRef) -> Package:
-        lookup = self._cache.get(ref)
-        if lookup is not None:
-            return lookup.archive.package
+        lookups = list(self._lookups(ref))
+        if len(lookups) == 1:
+            return lookups[0].archive.package
 
+        # TODO: More descriptive error message here
         raise PackageNotFoundError(ref)
 
     def data_type_name(self, ref: Any) -> TypeConName:
@@ -553,7 +575,7 @@ def template_names(self, ref: Any) -> Collection[TypeConName]:
         lookups = self._lookups(pkg)
 
         for lookup in lookups:
-            if name == "*":
+            if name == STAR:
                 names.extend(lookup.local_template_names())
             else:
                 n = lookup.local_template_name(name)
@@ -618,7 +640,7 @@ def template_or_interface_names(self, ref: Any) -> Collection[TypeConName]:
         lookups = self._lookups(pkg)
 
         for lookup in lookups:
-            if name == "*":
+            if name == STAR:
                 names.extend(lookup.local_template_names())
             else:
                 n = lookup.local_template_name(name)
@@ -669,12 +691,20 @@ def _lookups(self, ref: PackageRef) -> Iterable[PackageLookup]:
         :raises PackageNotFoundError:
             if the :class:`PackageRef` points to a package that is not present in this lookup.
         """
-        if ref == "*":
-            return self._cache.values()
+        if ref == STAR:
+            # unspecified package ID, so return all possible package lookups
+            return self._cache_by_pkg_id.values()
 
-        lookup = self._cache.get(ref)
-        if lookup is not None:
-            return (lookup,)
+        elif ref.startswith("#"):
+            # SCU-style template reference; find the most up-to-date package for this package name
+            lookup = self._cache_by_name.get(ref)
+            if lookup is not None:
+                return (lookup,)
+
+        else:
+            lookup = self._cache_by_pkg_id.get(ref)
+            if lookup is not None:
+                return (lookup,)
 
         raise PackageNotFoundError(ref)
 
@@ -760,3 +790,11 @@ def find_choice(template: DefTemplate, name: str) -> TemplateChoice:
             return choice
 
     raise NameNotFoundError(f"choice {name}")
+
+
+def get_version_tuple(lookup: PackageLookup) -> tuple[int, ...]:
+    md = lookup.archive.package.metadata
+    if md is not None:
+        return tuple([int(s) for s in md.version.split(".")])
+    else:
+        return ()

python/dazl/ledger/aio/pkgloader.py

@@ -14,7 +14,7 @@
 from ...damlast.errors import PackageNotFoundError
 from ...damlast.lookup import STAR, MultiPackageLookup, PackageExceptionTracker
 from ...damlast.parse import parse_archive
-from ...damlast.pkgfile import Dar
+from ...damlast.pkgfile import Dar, DarFile
 from ...prim import DazlError
 from ..auth import TokenOrTokenProvider
 
@@ -82,13 +82,15 @@ async def do_with_retry(
             with guard:
                 return fn()
 
-    async def preload(self, *contents: Dar) -> None:
+    async def preload(self, contents: bytes) -> None:
         """
-        Populate a :class:`PackageCache` with types from DARs.
+        Populate a :class:`PackageCache` with types from a DAR.
 
         :param contents:
-            One or more DARs to load into a local package cache.
+            A DAR to load in a local cache.
         """
+        with DarFile(contents) as dar_file:
+            self._package_lookup.add_archive(*dar_file.archives())
 
     async def load(
         self, ref: PackageRef, *, token: Optional[TokenOrTokenProvider] = None
@@ -104,9 +106,15 @@ async def load(
         :raises:
             PackageNotFoundError if the package could not be resolved
         """
-        if ref == STAR:
+        # With Daml 2, the best we can do when presented with a '*' or SCU-style template reference
+        # is to load _all_ packages, and give ourselves the best chance for finding the package we
+        # want. In Daml 3, newer services allow us to subscribe to package ID changes which will
+        # allow us to keep a local cache of packages more cheaply without the intermediate unnecessary
+        # fetching
+        if ref == STAR or ref.startswith("#"):
             await self.load_all(token=token)
-            return None
+            if ref == STAR:
+                return None
 
         # If the package has already been loaded, then skip all the expensive I/O stuff
         try:

python/dazl/ledger/grpc/codec_aio.py

@@ -29,7 +29,7 @@
     TypeConName,
 )
 from ...damlast.daml_types import ContractId as ContractIdType, con
-from ...damlast.lookup import MultiPackageLookup
+from ...damlast.lookup import MultiPackageLookup, validate_template
 from ...damlast.protocols import SymbolLookup, TemplateOrInterface
 from ...damlast.util import module_local_name, module_name, package_local_name, package_ref
 from ...ledger.aio import PackageService
@@ -92,6 +92,9 @@ def __init__(
     def lookup(self) -> SymbolLookup:
         return self._lookup
 
+    async def preload(self, contents) -> None:
+        await self._loader.preload(contents)
+
     async def encode_command(
         self, cmd: Command, /, *, token: Optional[TokenOrTokenProvider] = None
     ) -> lapipb.Command:
@@ -243,16 +246,21 @@ async def encode_filters(
         requested_templates = set[TypeConName]()
         requested_interfaces = set[TypeConName]()
         for template_or_interface_id in template_or_interface_ids:
-            requested_templates.update(
-                await self._loader.do_with_retry(
-                    lambda: self._lookup.template_names(template_or_interface_id), token=token
-                )
+            resolved_template_ids = await self._loader.do_with_retry(
+                lambda: self._lookup.template_names(template_or_interface_id), token=token
             )
-            requested_interfaces.update(
-                await self._loader.do_with_retry(
-                    lambda: self._lookup.interface_names(template_or_interface_id), token=token
-                )
+            if package_ref(template_or_interface_id) == "*":
+                requested_templates.update(resolved_template_ids)
+            elif resolved_template_ids:
+                requested_templates.add(template_or_interface_id)
+
+            resolved_interface_ids = await self._loader.do_with_retry(
+                lambda: self._lookup.interface_names(template_or_interface_id), token=token
             )
+            if package_ref(template_or_interface_id) == "*":
+                requested_interfaces.update(resolved_interface_ids)
+            elif resolved_interface_ids:
+                requested_interfaces.add(template_or_interface_id)
 
         return lapipb.Filters(
             inclusive=lapipb.InclusiveFilters(

python/dazl/ledger/grpc/conn_aio.py

@@ -1083,6 +1083,9 @@ async def upload_package(
                 lambda: stub.UploadDarFile(request, **call.grpc_kwargs), timeout=call.timeout
             )
 
+            # assuming the package has been uploaded, go ahead and load that locally too
+            await self._codec.preload(contents)
+
     # endregion
 
     # region Metering Report calls

python/tests/unit/dars.py

@@ -29,9 +29,7 @@ def load_dars() -> Mapping[str, Path]:
         if d.is_dir():
             with (d / "daml.yaml").open() as f:
                 daml_yaml = yaml.safe_load(f)
-            dars[daml_yaml["name"]] = (
-                d / f".daml/dist/{daml_yaml['name']}-{daml_yaml['version']}.dar"
-            )
+            dars[d.name] = d / f".daml/dist/{daml_yaml['name']}-{daml_yaml['version']}.dar"
 
     return dars
 
@@ -48,6 +46,8 @@ def load_dars() -> Mapping[str, Path]:
 MapSupport = DARS["map-support"]
 Pending = DARS["pending"]
 PostOffice = DARS["post-office"]
+SCUModels1 = DARS["scu-assets1"]
+SCUModels2 = DARS["scu-assets2"]
 Simple = DARS["simple"]
 TestServer = DARS["test-server"]
 UploadTest = DARS["upload-test"]

python/tests/unit/test_connection_command_submission.py

@@ -5,9 +5,10 @@
 
 import logging
 
+from dazl.ledger import CreateEvent
 from dazl.testing import SandboxLauncher, connect_with_new_party
 import pytest
-from tests.unit.dars import KitchenSink
+from tests.unit.dars import KitchenSink, SCUModels1, SCUModels2
 
 
 @pytest.mark.asyncio
@@ -24,6 +25,49 @@ async def test_create(sandbox: SandboxLauncher) -> None:
         )
 
 
+@pytest.mark.asyncio
+async def test_scu_create_and_query(sandbox: SandboxLauncher) -> None:
+    async with connect_with_new_party(url=sandbox.url, dar=SCUModels1, admin=True) as p:
+        issuer_party_info = await p.connection.allocate_party()
+        await p.connection.create(
+            "#AssetModels:Assets:Asset",
+            {
+                "issuer": issuer_party_info.party,
+                "owner": p.party,
+                "name": "Someone",
+            },
+            act_as=[issuer_party_info.party, p.party],
+        )
+
+        events_1 = list[CreateEvent]()
+        async with p.connection.query("#AssetModels:Assets:Asset") as stream:
+            async for event in stream.creates():
+                events_1.append(event)
+                logging.info("Found a contract after uploading 0.0.1: %s", event.payload)
+
+        await p.connection.upload_package(SCUModels2.read_bytes())
+
+        await p.connection.create(
+            "#AssetModels:Assets:Asset",
+            {
+                "issuer": issuer_party_info.party,
+                "owner": p.party,
+                "name": "Someone Else",
+                "desc": "A description",
+            },
+            act_as=[issuer_party_info.party, p.party],
+        )
+
+        events_2 = list[CreateEvent]()
+        async with p.connection.query("#AssetModels.Assets:Asset") as stream:
+            async for event in stream.creates():
+                events_2.append(event)
+                logging.info("Found a contract after uploading 0.0.2: %s", event.payload)
+
+        # we should now have two contracts
+        assert len(events_2) == 2
+
+
 @pytest.mark.asyncio
 async def test_exercise_by_key(sandbox: SandboxLauncher) -> None:
     async with connect_with_new_party(url=sandbox.url, dar=KitchenSink) as p:

python/tests/unit/test_validate_template.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2017-2025 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+from dazl.damlast.lookup import validate_template
+
+
+def test_scu_name():
+    pkg_ref, name = validate_template("#name:Module.SubModule:Template")
+    assert pkg_ref == "#name"
+    assert name == "Module.SubModule:Template"