Install a curated registry of crates with libraries

This largely mirrors the approach taken by major Linux distributions
like Ubuntu and Fedora. We can curate the crates in a way that should
allow us to use them in downstream package builds within this colcon
workspace or downstream workspaces.

This change doesn't yet add the code necessary to instruct cargo to use
our curated registry.

Author: cottsay

colcon_cargo/task/cargo/build.py

@@ -10,6 +10,8 @@
 from colcon_core.logging import colcon_logger
 from colcon_core.plugin_system import satisfies_version
 from colcon_core.shell import create_environment_hook, get_command_environment
+from colcon_core.task import create_file
+from colcon_core.task import install
 from colcon_core.task import run
 from colcon_core.task import TaskExtensionPoint
 
@@ -96,6 +98,11 @@ async def build(  # noqa: D102
             if rc and rc.returncode:
                 return rc.returncode
 
+        if self._has_libraries(metadata, pkg.name):
+            self.progress('package')
+            await self._install_package(
+                metadata['packages'][0]['version'], env)
+
         if not skip_hook_creation:
             create_environment_scripts(
                 pkg, args, additional_hooks=additional_hooks)
@@ -197,3 +204,82 @@ def _has_binaries(metadata, package_name):
         # If no binary target exists in the whole package, then skip running
         # cargo install because it would produce an error.
         return False
+
+    # Identify if there are any libraries to install for the current package
+    @staticmethod
+    def _has_libraries(metadata, package_name):
+        for package in metadata.get('packages', {}):
+            # If the package is part of a cargo workspace, the metadata
+            # contains all members. We're only interested in our target
+            # package - ignore the other workspace members here.
+            if package.get('name') != package_name:
+                continue
+            for target in package.get('targets', {}):
+                if {
+                    'lib',
+                    'rlib',
+                    'proc-macro',
+                }.intersection(target.get('crate_types', ())):
+                    # If any one binary exists in the package then we
+                    # should go ahead and install the extracted crate
+                    return True
+
+        # If no library target exists in the whole package, then skip extracted
+        # crate installation because it isn't useful.
+        return False
+
+    # Determine what files would be part of a packaged crate
+    async def _get_crate_contents(self, env):
+        pkg = self.context.pkg
+        cmd = [
+            CARGO_EXECUTABLE,
+            'package',
+            '--list',
+            '--allow-dirty',
+            '--quiet',
+            '--package', pkg.name,
+        ]
+
+        rc = await run(
+            self.context,
+            cmd,
+            cwd=self.context.pkg.path,
+            capture_output=True,
+            env=env
+        )
+        if rc is None or rc.returncode != 0:
+            raise RuntimeError(
+                "Could not inspect package using 'cargo package'"
+            )
+
+        if rc.stdout is None:
+            raise RuntimeError(
+                "Failed to capture stdout from 'cargo package'"
+            )
+
+        contents = set(rc.stdout.decode().splitlines())
+        contents.difference_update({
+            # Ignore stuff that we wouldn't want to copy
+            '',
+            None,
+            'Cargo.lock',
+            'Cargo.toml.orig',
+            '.cargo_vcs_info.json',
+        })
+        return contents
+
+    async def _install_package(self, version, env):
+        contents = await self._get_crate_contents(env)
+        crate_path = Path(
+            'share', 'cargo', 'registry', f'{self.context.pkg.name}-{version}')
+
+        for file in contents:
+            dst = crate_path / file
+            install(self.context.args, file, dst)
+
+        # Cargo "directory sources" require a checksum file to be included in
+        # the package metadata (though it need not list all of the files).
+        create_file(
+            self.context.args,
+            crate_path / '.cargo-checksum.json',
+            content='{"files":{},"package":""}\n')

test/spell_check.words

@@ -2,6 +2,7 @@ apache
 argcomplete
 asyncio
 autouse
+checksum
 colcon
 completers
 cwpd
@@ -26,6 +27,7 @@ pydocstyle
 pytest
 returncode
 rglob
+rlib
 rmtree
 rtype
 rustfmt

test/test_build.py

@@ -27,6 +27,7 @@
 TEST_PACKAGE_NAME = 'rust-sample-package'
 PURE_LIBRARY_PACKAGE_NAME = 'rust-pure-library'
 WORKSPACE_PACKAGE_NAME = 'rust-workspace'
+WORKSPACE_PACKAGE_VERSION = '0.1.0'
 
 test_project_path = Path(__file__).parent / TEST_PACKAGE_NAME
 pure_library_path = Path(__file__).parent / PURE_LIBRARY_PACKAGE_NAME
@@ -170,6 +171,7 @@ def test_build_and_test_package():
                                       path=str(test_project_path),
                                       build_base=str(tmpdir / 'build'),
                                       install_base=str(tmpdir / 'install'),
+                                      symlink_install=False,
                                       clean_build=None,
                                       cargo_args=None,
                                   ),
@@ -236,6 +238,7 @@ def test_skip_pure_library_package():
                                       path=str(pure_library_path),
                                       build_base=str(tmpdir / 'build'),
                                       install_base=str(tmpdir / 'install'),
+                                      symlink_install=False,
                                       clean_build=None,
                                       cargo_args=None,
                                   ),
@@ -290,6 +293,7 @@ def test_workspace_with_package():
                                       path=str(workspace_project_path),
                                       build_base=str(tmpdir / 'build'),
                                       install_base=str(tmpdir / 'install'),
+                                      symlink_install=False,
                                       clean_build=None,
                                       cargo_args=None,
                                   ),
@@ -315,5 +319,15 @@ def test_workspace_with_package():
             # members didn't get installed as well
             assert len(tuple((install_base / 'bin').iterdir())) == 1
 
+            # There should also be an unpacked library create
+            registry_path = install_base / 'share' / 'cargo' / 'registry'
+            crate_path = registry_path / '-'.join((
+                WORKSPACE_PACKAGE_NAME,
+                WORKSPACE_PACKAGE_VERSION,
+            ))
+            assert tuple(registry_path.iterdir()) == (crate_path,)
+            assert (crate_path / 'Cargo.toml').is_file()
+            assert (crate_path / 'src' / 'lib.rs').is_file()
+
     finally:
         event_loop.close()