Merge pull request #262 from digital-asset/python-daml-lf-1.14

python: Add support for Daml-LF 1.14 archives.

Author: da-tanabe

Makefile

@@ -2,13 +2,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 cache_dir=.cache
-daml_proto_version=1.9.0
+daml_proto_version=1.15.0-snapshot.20210705.7286.0.62aabcc4
 
 download_protos_zip := $(cache_dir)/download/protobufs-$(daml_proto_version).zip
-download_status_proto := $(cache_dir)/download/google/rpc/status.proto
 proto_dir := $(cache_dir)/protos
 proto_manifest := $(proto_dir)/manifest.json
 python := poetry run python3
+# TODO: Is there a programmatic way to get this path?
+venv_site_packages := .venv/lib/python3.6/site-packages
 
 version := $(shell python3 -c "import configparser; config = configparser.ConfigParser(); config.read('pyproject.toml'); print(config['tool.poetry']['version'][1:-1])")
 py_src := $(shell find python/dazl -name '*.py[i]') README.md pyproject.toml
@@ -49,16 +50,8 @@ $(download_protos_zip):
 	curl -sSL https://github.com/digital-asset/daml/releases/download/v$(daml_proto_version)/protobufs-$(daml_proto_version).zip -o $@
 
 
-$(download_status_proto):
-	@mkdir -p $(@D)
-	curl -sSL https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/status.proto -o $@
-
-
-$(proto_manifest): $(download_protos_zip) $(download_status_proto)
-	_build/unpack.py \
-	  -i $(download_protos_zip) \
-	  -i $(download_status_proto) \
-	  -o $(@D) -m $@
+$(proto_manifest): $(download_protos_zip)
+	_build/unpack.py -i $^ -o $(@D) -m $@
 
 
 .PHONY: unpack-protos

VERSION

@@ -1 +1 @@
-7.5.5
+7.5.6

_build/python/make-fragment

@@ -81,7 +81,7 @@ def generate(manifest_file):
             print()
             print(f'$(_py_gen_tmp_dir)/{p}_pb2.py: $(proto_dir)/{p}.proto .venv/poetry.lock')
             print('\t@mkdir -p $(_py_gen_tmp_dir)')
-            print('\t$(python) -m grpc_tools.protoc -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) $<')
+            print('\t$(python) -m grpc_tools.protoc -I$(venv_site_packages) -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) $<')
 
         if 'grpc' in types:
             print()
@@ -95,7 +95,7 @@ def generate(manifest_file):
             print()
             print(f'$(_py_gen_tmp_dir)/{p}_pb2.py $(_py_gen_tmp_dir)/{p}_pb2_grpc.py: $(proto_dir)/{p}.proto COPYRIGHT')
             print('\t@mkdir -p $(_py_gen_tmp_dir)')
-            print('\t$(python) -m grpc_tools.protoc -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) --grpc_python_out=$(_py_gen_tmp_dir) $<')
+            print('\t$(python) -m grpc_tools.protoc -I$(venv_site_packages) -I$(proto_dir) --python_out=$(_py_gen_tmp_dir) --grpc_python_out=$(_py_gen_tmp_dir) $<')
 
 
 if __name__ == '__main__':

_build/unpack.py

@@ -4,9 +4,10 @@
 
 import json
 import logging
+import os
 from os.path import basename
 from pathlib import Path
-from typing import Mapping, Sequence
+from typing import Mapping, Sequence, Union, Dict
 from zipfile import ZipFile
 from io import TextIOWrapper
 
@@ -19,7 +20,7 @@ def main():
 
     logging.basicConfig(level=logging.INFO)
 
-    parser = ArgumentParser(description='Unpack the DAML SDK protobuf package.')
+    parser = ArgumentParser(description='Unpack the Daml Connect Protobuf package.')
     parser.add_argument('--input', '-i', required=True, action='append')
     parser.add_argument('--output', '-o', required=True)
     parser.add_argument('--output-manifest', '-m')
@@ -48,83 +49,60 @@ def run(self) -> 'Manifest':
         for input_path in self.inputs:
             if input_path.suffix == '.zip':
                 manifest.update(self._process_zip(input_path))
-            elif input_path.suffix == '.proto':
-                manifest.update(self._process_proto(input_path))
             else:
-                raise ValueError(f"don't know how to process {input_path.path}")
+                raise ValueError(f"don't know how to process {input_path}")
 
         return manifest
 
     def _process_zip(self, path: 'Path') -> 'Manifest':
-        proto_packages = {}  # type: Manifest
+        proto_packages = {}  # type: Dict[str, Sequence[str]]
         with ZipFile(path) as z:
-            logging.info('Procesing zip file: %s', path)
+            logging.info('Processing zip file: %s', path)
             for zi in z.infolist():
                 logging.info('    Processing file: %s', zi.filename)
                 if is_valid_proto(zi.filename) and not zi.is_dir():
+                    # strip off the top-level directory name; we don't need it
+                    _, _, path = zi.filename.partition('/')
                     with z.open(zi) as f:
                         with TextIOWrapper(f) as text_buf:
                             contents = text_buf.read()
                     try:
-                        proto_packages.update(self._process_proto_contents(basename(zi.filename), contents))
+                        proto_packages.update(self._process_proto_contents(path, contents))
                     except ValueError:
                         logging.exception('Could not process %s', zi.filename)
         return proto_packages
 
-    def _process_proto(self, path: 'Path') -> 'Manifest':
-        try:
-            return self._process_proto_contents(path.name, path.read_text())
-        except ValueError:
-            logging.exception('Could not process %s', path)
-        return {}
-
-    def _process_proto_contents(self, short_name: str, contents: str) -> 'Manifest':
+    def _process_proto_contents(self, rel_file: str, contents: str) -> 'Manifest':
         lines = contents.splitlines()
         if not lines:
             return {}
 
-        # look for a package directive
-        package_lines = [line for line in lines if line.startswith('package ')]
-        if not package_lines:
-            raise ValueError('file is missing a package directive')
-
-        package = package_lines[0].rstrip(';')[8:]
-
         is_grpc = any(line.startswith('service') for line in lines)
 
-        rel_file = dest_dir(package) + '/' + short_name
-        
         out_file = self.output / rel_file
         out_file.parent.mkdir(parents=True, exist_ok=True)
         out_file.write_text(contents)
 
         return { rel_file.rpartition('.')[0]: ['pb', 'grpc'] if is_grpc else ['pb'] }
 
 
-def dest_dir(package: str) -> str:
-    """
-    Modify the package directive in a Protobuf file to put it in a more "expected" place.
-    """
-    if package in ('daml_lf_dev', 'daml_lf_1'):
-        return 'com/daml/daml_lf_dev'
-    else:
-        return package.replace('.', '/')
-
-
-def is_valid_proto(path: 'Path') -> bool:
+def is_valid_proto(path: 'Union[str, Path]') -> bool:
     # We prefer DAML-LF 1.dev because dev is always a strict superset of
     # the latest 1.x version, but also allows us to build and test support for
     # experimental features before they are released.
     if path is None:
         return False
 
-    p = str(path)
+    p = os.fspath(path)
     return p.endswith('.proto') and \
         '/daml_lf_1_6/' not in p and \
         '/daml_lf_1_7/' not in p and \
         '/daml_lf_1_8/' not in p and \
-        '/daml_lf_1_11/' not in p
-    
+        '/daml_lf_1_11/' not in p and \
+        '/daml_lf_1_12/' not in p and \
+        '/daml_lf_1_13/' not in p and \
+        '/daml_lf_dev/' not in p
+
 
 if __name__ == '__main__':
     main()

pyproject.toml

@@ -3,7 +3,7 @@
 
 [tool.poetry]
 name = "dazl"
-version = "7.5.5"
+version = "7.5.6"
 description = "high-level Ledger API client for DAML ledgers"
 license = "Apache-2.0"
 authors = ["Davin K. Tanabe <davin.tanabe@digitalasset.com>"]

…zl/_gen/com/daml/daml_lf_dev/__init__.py …l/_gen/com/daml/daml_lf_1_14/__init__.pypython/dazl/_gen/com/daml/daml_lf_dev/__init__.py renamed to python/dazl/_gen/com/daml/daml_lf_1_14/__init__.py python/dazl/_gen/com/daml/daml_lf_dev/__init__.py renamed to python/dazl/_gen/com/daml/daml_lf_1_14/__init__.py

@@ -2,8 +2,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: com/daml/daml_lf_dev/daml_lf.proto
-
+# source: com/daml/daml_lf_1_14/daml_lf.proto
+"""Generated protocol buffer code."""
 from google.protobuf import (
     descriptor as _descriptor,
     message as _message,
@@ -17,23 +17,23 @@
 _sym_db = _symbol_database.Default()
 
 
-from . import daml_lf_1_pb2 as com_dot_daml_dot_daml__lf__dev_dot_daml__lf__1__pb2
+from . import daml_lf_1_pb2 as com_dot_daml_dot_daml__lf__1__14_dot_daml__lf__1__pb2
 
 DESCRIPTOR = _descriptor.FileDescriptor(
-    name="com/daml/daml_lf_dev/daml_lf.proto",
-    package="daml_lf_dev",
+    name="com/daml/daml_lf_1_14/daml_lf.proto",
+    package="daml_lf_1_13",
     syntax="proto3",
-    serialized_options=b"\n\024com.daml.daml_lf_dev\252\002\033Com.Daml.Daml_Lf_Dev.DamlLf",
+    serialized_options=b"\n\025com.daml.daml_lf_1_14\252\002\034Com.Daml.Daml_Lf_1_14.DamlLf",
     create_key=_descriptor._internal_create_key,
-    serialized_pb=b'\n"com/daml/daml_lf_dev/daml_lf.proto\x12\x0b\x64\x61ml_lf_dev\x1a$com/daml/daml_lf_dev/daml_lf_1.proto"]\n\x0e\x41rchivePayload\x12\r\n\x05minor\x18\x03 \x01(\t\x12\'\n\tdaml_lf_1\x18\x02 \x01(\x0b\x32\x12.daml_lf_1.PackageH\x00\x42\x05\n\x03SumJ\x06\x08\x8fN\x10\x90NJ\x04\x08\x01\x10\x02"Z\n\x07\x41rchive\x12\x30\n\rhash_function\x18\x01 \x01(\x0e\x32\x19.daml_lf_dev.HashFunction\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x0c\n\x04hash\x18\x04 \x01(\t*\x1a\n\x0cHashFunction\x12\n\n\x06SHA256\x10\x00\x42\x34\n\x14\x63om.daml.daml_lf_dev\xaa\x02\x1b\x43om.Daml.Daml_Lf_Dev.DamlLfb\x06proto3',
+    serialized_pb=b'\n#com/daml/daml_lf_1_14/daml_lf.proto\x12\x0c\x64\x61ml_lf_1_13\x1a%com/daml/daml_lf_1_14/daml_lf_1.proto"]\n\x0e\x41rchivePayload\x12\r\n\x05minor\x18\x03 \x01(\t\x12\'\n\tdaml_lf_1\x18\x02 \x01(\x0b\x32\x12.daml_lf_1.PackageH\x00\x42\x05\n\x03SumJ\x06\x08\x8fN\x10\x90NJ\x04\x08\x01\x10\x02"[\n\x07\x41rchive\x12\x31\n\rhash_function\x18\x01 \x01(\x0e\x32\x1a.daml_lf_1_13.HashFunction\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x0c\n\x04hash\x18\x04 \x01(\t*\x1a\n\x0cHashFunction\x12\n\n\x06SHA256\x10\x00\x42\x36\n\x15\x63om.daml.daml_lf_1_14\xaa\x02\x1c\x43om.Daml.Daml_Lf_1_14.DamlLfb\x06proto3',
     dependencies=[
-        com_dot_daml_dot_daml__lf__dev_dot_daml__lf__1__pb2.DESCRIPTOR,
+        com_dot_daml_dot_daml__lf__1__14_dot_daml__lf__1__pb2.DESCRIPTOR,
     ],
 )
 
 _HASHFUNCTION = _descriptor.EnumDescriptor(
     name="HashFunction",
-    full_name="daml_lf_dev.HashFunction",
+    full_name="daml_lf_1_13.HashFunction",
     filename=None,
     file=DESCRIPTOR,
     create_key=_descriptor._internal_create_key,
@@ -49,8 +49,8 @@
     ],
     containing_type=None,
     serialized_options=None,
-    serialized_start=276,
-    serialized_end=302,
+    serialized_start=280,
+    serialized_end=306,
 )
 _sym_db.RegisterEnumDescriptor(_HASHFUNCTION)
 
@@ -60,15 +60,15 @@
 
 _ARCHIVEPAYLOAD = _descriptor.Descriptor(
     name="ArchivePayload",
-    full_name="daml_lf_dev.ArchivePayload",
+    full_name="daml_lf_1_13.ArchivePayload",
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
             name="minor",
-            full_name="daml_lf_dev.ArchivePayload.minor",
+            full_name="daml_lf_1_13.ArchivePayload.minor",
             index=0,
             number=3,
             type=9,
@@ -87,7 +87,7 @@
         ),
         _descriptor.FieldDescriptor(
             name="daml_lf_1",
-            full_name="daml_lf_dev.ArchivePayload.daml_lf_1",
+            full_name="daml_lf_1_13.ArchivePayload.daml_lf_1",
             index=1,
             number=2,
             type=11,
@@ -115,29 +115,29 @@
     oneofs=[
         _descriptor.OneofDescriptor(
             name="Sum",
-            full_name="daml_lf_dev.ArchivePayload.Sum",
+            full_name="daml_lf_1_13.ArchivePayload.Sum",
             index=0,
             containing_type=None,
             create_key=_descriptor._internal_create_key,
             fields=[],
         ),
     ],
-    serialized_start=89,
-    serialized_end=182,
+    serialized_start=92,
+    serialized_end=185,
 )
 
 
 _ARCHIVE = _descriptor.Descriptor(
     name="Archive",
-    full_name="daml_lf_dev.Archive",
+    full_name="daml_lf_1_13.Archive",
     filename=None,
     file=DESCRIPTOR,
     containing_type=None,
     create_key=_descriptor._internal_create_key,
     fields=[
         _descriptor.FieldDescriptor(
             name="hash_function",
-            full_name="daml_lf_dev.Archive.hash_function",
+            full_name="daml_lf_1_13.Archive.hash_function",
             index=0,
             number=1,
             type=14,
@@ -156,7 +156,7 @@
         ),
         _descriptor.FieldDescriptor(
             name="payload",
-            full_name="daml_lf_dev.Archive.payload",
+            full_name="daml_lf_1_13.Archive.payload",
             index=1,
             number=3,
             type=12,
@@ -175,7 +175,7 @@
         ),
         _descriptor.FieldDescriptor(
             name="hash",
-            full_name="daml_lf_dev.Archive.hash",
+            full_name="daml_lf_1_13.Archive.hash",
             index=2,
             number=4,
             type=9,
@@ -201,13 +201,13 @@
     syntax="proto3",
     extension_ranges=[],
     oneofs=[],
-    serialized_start=184,
-    serialized_end=274,
+    serialized_start=187,
+    serialized_end=278,
 )
 
 _ARCHIVEPAYLOAD.fields_by_name[
     "daml_lf_1"
-].message_type = com_dot_daml_dot_daml__lf__dev_dot_daml__lf__1__pb2._PACKAGE
+].message_type = com_dot_daml_dot_daml__lf__1__14_dot_daml__lf__1__pb2._PACKAGE
 _ARCHIVEPAYLOAD.oneofs_by_name["Sum"].fields.append(_ARCHIVEPAYLOAD.fields_by_name["daml_lf_1"])
 _ARCHIVEPAYLOAD.fields_by_name["daml_lf_1"].containing_oneof = _ARCHIVEPAYLOAD.oneofs_by_name["Sum"]
 _ARCHIVE.fields_by_name["hash_function"].enum_type = _HASHFUNCTION
@@ -221,8 +221,8 @@
     (_message.Message,),
     {
         "DESCRIPTOR": _ARCHIVEPAYLOAD,
-        "__module__": "com.daml.daml_lf_dev.daml_lf_pb2"
-        # @@protoc_insertion_point(class_scope:daml_lf_dev.ArchivePayload)
+        "__module__": "com.daml.daml_lf_1_14.daml_lf_pb2"
+        # @@protoc_insertion_point(class_scope:daml_lf_1_13.ArchivePayload)
     },
 )
 _sym_db.RegisterMessage(ArchivePayload)
@@ -232,8 +232,8 @@
     (_message.Message,),
     {
         "DESCRIPTOR": _ARCHIVE,
-        "__module__": "com.daml.daml_lf_dev.daml_lf_pb2"
-        # @@protoc_insertion_point(class_scope:daml_lf_dev.Archive)
+        "__module__": "com.daml.daml_lf_1_14.daml_lf_pb2"
+        # @@protoc_insertion_point(class_scope:daml_lf_1_13.Archive)
     },
 )
 _sym_db.RegisterMessage(Archive)