fix(apt-cache): also parse unsigned apt cache release files

src/debsbom/apt/cache.py

@@ -58,7 +58,7 @@ def from_file(
 class Repository:
     """Represents a debian repository as cached by apt."""
 
-    in_release_file: Path
+    release_file: Path
     origin: str | None
     codename: str | None
     architectures: list[str]
@@ -73,7 +73,7 @@ class Repository:
     def from_apt_cache(cls, lists_dir: str | Path) -> Iterable["Repository"]:
         """Create repositories from apt lists directory."""
         for entry in Path(lists_dir).iterdir():
-            if entry.name.endswith("_InRelease"):
+            if entry.name.endswith("Release"):
                 with open(entry) as f:
                     repo = Deb822(f)
                 origin = repo.get("Origin")
@@ -87,7 +87,7 @@ def from_apt_cache(cls, lists_dir: str | Path) -> Iterable["Repository"]:
                     logger.error(f"Repository does not specify 'Architectures', ignoring: {entry}")
                     continue
                 yield Repository(
-                    in_release_file=entry,
+                    release_file=entry,
                     origin=origin,
                     codename=codename,
                     version=Version(version) if version else None,
@@ -201,17 +201,20 @@ def _parse_packages(
         except (FileNotFoundError, IndexError, RuntimeError):
             logger.debug(f"Missing apt cache packages: {packages_file}")
 
+    @property
+    def repo_base(self):
+        return "_".join(str(self.release_file).split("_")[:-1])
+
     def sources(
         self, filter_fn: Callable[[SourcePackageFilter], bool] | None = None
     ) -> Iterable[SourcePackage]:
         """Get all source packages from this repository."""
-        repo_base = str(self.in_release_file).removesuffix("_InRelease")
         if self.components:
             for component in self.components:
-                sources_file = "_".join([repo_base, component, "source", "Sources"])
+                sources_file = "_".join([self.repo_base, component, "source", "Sources"])
                 yield from self._parse_sources(sources_file, filter_fn)
         else:
-            sources_file = "_".join([repo_base, "source", "Sources"])
+            sources_file = "_".join([self.repo_base, "source", "Sources"])
             return self._parse_sources(sources_file, filter_fn)
 
     def binpackages(
@@ -220,17 +223,18 @@ def binpackages(
         ext_states: ExtendedStates = ExtendedStates(set()),
     ) -> Iterable[BinaryPackage]:
         """Get all binary packages from this repository"""
-        repo_base = str(self.in_release_file).removesuffix("_InRelease")
         if self.components:
             for component in self.components:
                 for arch in self.architectures:
-                    packages_file = "_".join([repo_base, component, f"binary-{arch}", "Packages"])
+                    packages_file = "_".join(
+                        [self.repo_base, component, f"binary-{arch}", "Packages"]
+                    )
                     for p in self._parse_packages(packages_file, filter_fn):
                         p.manually_installed = ext_states.is_manual(p.name, p.architecture)
                         yield p
         else:
             for arch in self.architectures:
-                packages_file = "_".join([repo_base, f"binary-{arch}", "Packages"])
+                packages_file = "_".join([self.repo_base, f"binary-{arch}", "Packages"])
                 for p in self._parse_packages(packages_file, filter_fn):
                     p.manually_installed = ext_states.is_manual(p.name, p.architecture)
                     yield p

src/debsbom/dpkg/package.py

@@ -383,11 +383,8 @@ def _resolve_sources(cls, pkg: "BinaryPackage", add_pkg=False) -> Iterable["Pack
             logger.debug(f"Found source package: '{pkg.source.name}'")
             yield src_pkg
         for bu in pkg.built_using:
-            # When creating the source package from a built-depends, we don't know the maintainer.
-            # If we now create a source package first via a built-using relation and later
-            # re-create the same source package from a binary package, it still misses the
-            # maintainer information, despite we would have it from the binary package.
-            # Some tests on a rather large debian sid showed, that this situation is unlikely.
+            # Add partial source package. This can later be enhanced by merging it with
+            # a more complete source package we discovered via other mechanisms (e.g. apt cache).
             logger.debug(f"Found built-using source package: '{bu.name}@{bu.version[1]}'")
             yield SourcePackage(bu.name, bu.version[1])
         if add_pkg:

src/debsbom/generate/generate.py

@@ -245,6 +245,10 @@ def binary_filter(bpf: Repository.BinaryPackageFilter) -> bool:
                 shash = hash(source_pkg)
                 if shash not in packages:
                     to_add.append(source_pkg)
+                else:
+                    # at this point in time we already have the apt data, so we merge our
+                    # incomplete source packages with the proper ones from apt.
+                    packages[shash].merge_with(source_pkg)
             # we add it in a separate loop so we do not invalidate the packages iterator
             for source_pkg in to_add:
                 packages[hash(source_pkg)] = source_pkg

tests/root/apt-sources/var/lib/apt/lists/local.example.com_dists_stable_Release

@@ -0,0 +1,20 @@
+Origin: Local
+Label: Local Repository
+Suite: stable
+Version: 1.0
+Codename: stable
+Architectures: amd64
+Components: main
+Description: Local unsigned repository
+MD5Sum:
+ 8c84058db82470c3eb3db5f558d2fb61 519316119 main/Contents-all
+ 4d583fc44d208a68fb53f5f74e11c07e 50065615 main/binary-amd64/Packages
+ 669c6f8f59f5d00072605a65f0e5950f      120 main/binary-amd64/Release
+ 75f089b57bf67e60c9f5e06181cf563b      121 main/source/Release
+ 17c3c2cebfd99cb4a6501e2bc7ad7f65   209044 main/source/Sources
+SHA256:
+ d6c9c82f4e61b4662f9ba16b9ebb379c57b4943f8b7813091d1f637325ddfb79  1484322 main/Contents-all
+ 2cc832a2786983db27c46360a7671610c3d1667994e20d84c1ba2c1f16e1e8b4 50065615 main/binary-amd64/Packages
+ 1751a0ddff790494b681cf4ab713c2eb6017dec70d1349e5ffe94f7bc3e71f9a      120 main/binary-amd64/Release
+ 6ac5f49194a88b97c29e4bba555cff3e8dfea5849ed63e4e6a70c9289d68ef28      121 main/source/Release
+ 17e57309b0ca2fafa552e1ad0b6fcecb237ac8709ac28eb0bc1fb293544934dd   209044 main/source/Sources

tests/root/apt-sources/var/lib/apt/lists/local.example.com_dists_stable_main_binary-amd64_Packages

@@ -0,0 +1 @@
+deb.debian.org_debian_dists_bookworm_main_binary-amd64_Packages

tests/root/apt-sources/var/lib/apt/lists/local.example.com_dists_stable_main_source_Sources

@@ -0,0 +1 @@
+deb.debian.org_debian_dists_bookworm_main_source_Sources

tests/test_generation.py

@@ -295,10 +295,12 @@ def test_apt_extended_states():
     assert noes.is_manual("foo", "amd64")
 
 
-def test_apt_cache_parsing():
+@pytest.mark.parametrize("origin", ["Debian", "Local"])
+def test_apt_cache_parsing(origin):
     apt_lists_dir = "tests/root/apt-sources/var/lib/apt/lists"
-    repo = next(Repository.from_apt_cache(apt_lists_dir))
-    src_pkgs = list(repo.sources(lambda p: p.name == "binutils"))
+    repos = list(Repository.from_apt_cache(apt_lists_dir))
+    deb_repo = next(filter(lambda r: r.origin == origin, repos))
+    src_pkgs = list(deb_repo.sources(lambda p: p.name == "binutils"))
     assert len(src_pkgs) == 1
     # this data is only available in apt sources deb822 data
     assert "binutils-for-host" in src_pkgs[0].binaries