vector_vendor: checked-in binary, no network fetch

.github/workflows/ci.yaml

@@ -18,9 +18,9 @@
 #   kpi-views (standalone: SQL + a throwaway Postgres, no workspace image needed)
 #   raw-volume (standalone: stdlib Python fixtures over the shared volume accounting)
 #   network-isolation-check (standalone: builds tools/e2e/Containerfile's
-#     vendor-network-check stage — #423. Disabled (if: false) until #424/#425 replace
-#     aws_sdk_vendor's/vector_vendor's build-time network fetches with vendored/prebuilt
-#     sources — until then this job's build is *expected* to fail.)
+#     vendor-network-check stage — #423. vector_vendor's build-time fetch is already
+#     replaced with a checked-in binary (#424); disabled (if: false) until #425 does
+#     the same for aws_sdk_vendor's — until then this job's build is *expected* to fail.)
 #
 # e2e calls the same tools/e2e/scripts/run.sh a developer runs locally, driving the
 # harness with plain podman (no compose) so nothing extra is installed on the runner;
@@ -339,11 +339,10 @@ jobs:
   # continue-on-error: true — the job's own check-run conclusion reports "failure"
   # regardless of continue-on-error (only the *workflow's* aggregate conclusion and
   # `needs:`-blocking are affected by it), so every PR still showed a red ❌ for a
-  # failure that's expected today, not actionable. aws_sdk_vendor still git-clones
-  # aws-sdk-cpp and vector_vendor still downloads a prebuilt binary, both at
-  # colcon-build time, until #424/#425 replace those fetches with vendored/prebuilt
-  # sources — flip `if: false` to `if: true` (and drop this comment) once they land
-  # and this build can actually pass.
+  # failure that's expected today, not actionable. vector_vendor now installs a
+  # checked-in binary (#424), but aws_sdk_vendor still git-clones aws-sdk-cpp at
+  # colcon-build time — flip `if: false` to `if: true` (and drop this comment) once
+  # #425 replaces that fetch too and this build can actually pass.
   network-isolation-check:
     runs-on: ubuntu-latest
     if: false
@@ -353,7 +352,7 @@ jobs:
       - name: Verify podman
         run: podman --version
 
-      - name: Attempt the network-isolated vendor package build (expected to fail until #424/#425 land)
+      - name: Attempt the network-isolated vendor package build (expected to fail until #425 lands)
         run: ./tools/e2e/scripts/verify_network_isolation.sh
 
   # The KPI definitions are SQL, not workspace code: this needs a database and nothing else.

.pre-commit-config.yaml

@@ -23,6 +23,9 @@ repos:
       - id: check-added-large-files
         name: Detect if giant files are added
         args: ["--maxkb=2000"]
+        # vector_vendor's checked-in Vector release tarballs (#424) replace a
+        # build-time network download with a committed binary by design.
+        exclude: "^vector_vendor/prebuilt/.*\\.tar\\.gz$"
       - id: check-docstring-first
         name: Checks for a common error of placing code before the docstring
       - id: check-executables-have-shebangs

REUSE.toml

@@ -119,3 +119,12 @@ path = [
 ]
 SPDX-FileCopyrightText = "2012 Nathan Koenig"
 SPDX-License-Identifier = "CC-BY-3.0"
+
+# --- Vendored third-party binaries ---------------------------------------------------
+
+# Official vectordotdev/vector release tarballs, checked in verbatim (#424) so
+# vector_vendor's build needs no network access; see vector_vendor/CMakeLists.txt.
+[[annotations]]
+path = "vector_vendor/prebuilt/*.tar.gz"
+SPDX-FileCopyrightText = "2020 Vector Authors <vector@datadoghq.com>"
+SPDX-License-Identifier = "MPL-2.0"

doc/src/dc/migration.md

@@ -583,9 +583,10 @@ dc_bridge:
   root to run it). `fluent_bit_plugins` and `dc_destinations` no longer exist.
 - **No Go toolchain.** The `out_minio.so` / `out_files_metrics.so` Go plugins are gone
   along with `plugin_path`.
-- **Vector is vendored.** `vector_vendor` downloads a pinned, checksummed Vector binary
-  at build time. Point `-Dvector_path=/usr/bin/vector` (or `VECTOR_PATH`) at a
-  system-installed binary for air-gapped builds.
+- **Vector is vendored.** `vector_vendor` installs a pinned, checksummed Vector binary
+  checked directly into the package — no network access at build time. Point
+  `-Dvector_path=/usr/bin/vector` (or `VECTOR_PATH`) at a system-installed binary
+  instead if preferred.
 - **Startup is ordered.** The Bridge starts before the collection nodes and a readiness
   gate blocks activation until the Shipper is accepting connections, so no Record is
   produced before the pipeline can accept it. See

doc/src/dc/setup.md

@@ -35,12 +35,12 @@ source /opt/ros/jazzy/setup.bash
 colcon build
 ```
 
-That is the whole install. `colcon build` also runs `vector_vendor`, which downloads a
-pinned, checksummed [Vector](https://vector.dev/) binary — the external **Shipper** the
-Bridge supervises at runtime (ADR-0002).
+That is the whole install. `colcon build` also runs `vector_vendor`, which installs a
+pinned, checksummed [Vector](https://vector.dev/) binary checked into the package — the
+external **Shipper** the Bridge supervises at runtime (ADR-0002).
 
 ```admonish tip title="Air-gapped or distro-packaged Vector"
-Point the build at a Vector binary you already have instead of downloading one:
+Point the build at a Vector binary you already have instead of the checked-in one:
 
     colcon build --cmake-args -Dvector_path=/usr/bin/vector
 

docs/adr/0002-vector-as-default-shipper.md

@@ -4,8 +4,46 @@ Both upstream Fluent Bit and Vector can sit behind the Bridge's Forward-protocol
 
 ## Consequences
 
-- CI publishes a "vector-slim" prebuilt binary (feature-flagged build, ~30–40MB) for constrained deployments; users never compile Vector themselves.
-- A `vector_vendor` ament package downloads the official static binary at build time, pinned to an exact version by checksum (arch-detected); a `vector_path` parameter allows using a system-installed Vector instead. Apt-repo install and Docker remain documented alternatives.
+- A `vector_vendor` ament package vendors the official static binary, pinned to an exact version by checksum (arch-detected); a `vector_path` parameter allows using a system-installed Vector instead. Apt-repo install and Docker remain documented alternatives.
+
+## Amendment: checked-in binary instead of a build-time download (#424)
+
+`vector_vendor` originally fetched its pinned binary over the network at build time via
+`file(DOWNLOAD ...)`, which violates the ROS buildfarm's no-network-access policy for
+binarydeb jobs and defeats #423's network-isolated build check. The "vector-slim"
+feature-flagged build this ADR originally proposed CI would publish was never actually
+built — no such CI job exists in this repo — so it is not available to vendor either.
+
+Fixed by checking the official `vectordotdev/vector` release tarballs (x86_64 and
+aarch64, same bytes `file(DOWNLOAD ...)` used to fetch, same pinned SHA256 checksums)
+directly into `vector_vendor/prebuilt/`; `CMakeLists.txt` now extracts and installs from
+the local file, performing no network I/O. The *tarball*, not the extracted binary, is
+what's committed: the extracted `vector` binary is 120–142MB (stripped/unstripped)
+depending on target, over GitHub's 100MB hard per-file limit, while the release
+tarball's upstream gzip compression brings each architecture's file to ~50-54MB — under
+the limit.
+
+**Not tracked via Git LFS**, per #421's original implementation decision ("no
+release-time injection step, no `git-lfs`: committed like any other tracked file").
+LFS was evaluated and briefly adopted mid-review: the specific concern that motivated
+#421's original call — that LFS content wouldn't survive `bloom`'s release-tarball
+export step — turned out to be unfounded. `bloom`'s `export_upstream` delegates to
+`vcstools.GitClient.checkout()` (a plain `git clone` + `git checkout <tag>`, which does
+run LFS's smudge filter), and the export machinery itself
+(`vcstools.git_archive_all.GitArchiver`) reads files off the working-tree filesystem
+rather than through `git archive`'s blob-store plumbing (the actual, narrower reason
+plain `git archive` breaks LFS) — confirmed empirically against a throwaway LFS repo and
+the real `vcstools`/`bloom` code, not just read.
+
+LFS was reverted anyway for a more basic reason: GitHub's free LFS tier is 1GB storage
+and 1GB bandwidth per month, and every future Vector version bump adds both tarballs
+(~106MB) as new, non-deduplicated LFS objects — roughly 9 version bumps before storage
+alone exhausts the free tier, independent of and sooner than the plain-blob approach's
+own cost (git history growing by the same ~106MB per bump, but against no comparable
+quota). A plain committed blob has no such ceiling; the tradeoff is that every future
+Vector bump grows this repo's ordinary git history by ~106MB, permanently, since git
+does not deduplicate binary blobs across versions. Revisit if/when that accumulation
+becomes the more pressing cost — the LFS path is proven to work, should it be needed.
 
 ## Why the forward protocol is the Bridge→Shipper wire format
 

vector_vendor/CMakeLists.txt

@@ -6,8 +6,9 @@ project(vector_vendor)
 
 find_package(ament_cmake REQUIRED)
 
-# Bumping Vector is a single-line change: update the version and the checksum
-# for every supported architecture below (see ADR-0002).
+# Bumping Vector: download the new version's release tarball for every architecture
+# below, drop it into prebuilt/ (replacing the old one), and update the version and
+# checksum here to match (see ADR-0002).
 set(VECTOR_VERSION "0.57.0")
 set(VECTOR_SHA256_x86_64 "b04fafcffcbeb3d43cbba587583dbe3338b24dd34256b3967fb15d137aff75f2")
 set(VECTOR_SHA256_aarch64 "81cec5343e49985351e13f76ae3d550d7b0f0ac64fd4d88c07e3873bdfdf100a")
@@ -44,36 +45,22 @@ else()
     message(FATAL_ERROR "vector_vendor: no pinned checksum for architecture '${_vector_arch}'")
   endif()
 
+  # The official release tarball (same bytes `file(DOWNLOAD ...)` used to fetch, same
+  # pinned checksum below) is checked directly into this package's git history instead
+  # of fetched over the network at build time (#424) — no network access required.
   set(_vector_tarball_name "vector-${VECTOR_VERSION}-${_vector_arch}-unknown-linux-musl.tar.gz")
-  set(_vector_url "https://github.com/vectordotdev/vector/releases/download/v${VECTOR_VERSION}/${_vector_tarball_name}")
-  set(_vector_download_dir "${CMAKE_CURRENT_BINARY_DIR}/download")
-  set(_vector_tarball "${_vector_download_dir}/${_vector_tarball_name}")
-
-  file(MAKE_DIRECTORY "${_vector_download_dir}")
-
-  set(_need_download TRUE)
-  if(EXISTS "${_vector_tarball}")
-    file(SHA256 "${_vector_tarball}" _cached_sha256)
-    if(_cached_sha256 STREQUAL _vector_sha256)
-      set(_need_download FALSE)
-    endif()
+  set(_vector_tarball "${CMAKE_CURRENT_SOURCE_DIR}/prebuilt/${_vector_tarball_name}")
+  if(NOT EXISTS "${_vector_tarball}")
+    message(FATAL_ERROR
+      "vector_vendor: no checked-in tarball for architecture '${_vector_arch}' at "
+      "${_vector_tarball}")
   endif()
 
-  if(_need_download)
-    message(STATUS "vector_vendor: downloading ${_vector_url}")
-    file(DOWNLOAD "${_vector_url}" "${_vector_tarball}"
-      EXPECTED_HASH SHA256=${_vector_sha256}
-      TLS_VERIFY ON
-      STATUS _download_status
-    )
-    list(GET _download_status 0 _download_code)
-    if(NOT _download_code EQUAL 0)
-      list(GET _download_status 1 _download_message)
-      file(REMOVE "${_vector_tarball}")
-      message(FATAL_ERROR
-        "vector_vendor: failed to download Vector ${VECTOR_VERSION} for "
-        "${_vector_arch}: ${_download_message}")
-    endif()
+  file(SHA256 "${_vector_tarball}" _actual_sha256)
+  if(NOT _actual_sha256 STREQUAL _vector_sha256)
+    message(FATAL_ERROR
+      "vector_vendor: checked-in tarball '${_vector_tarball}' has SHA256 "
+      "${_actual_sha256}, expected ${_vector_sha256}")
   endif()
 
   set(_vector_extract_dir "${CMAKE_CURRENT_BINARY_DIR}/extract")
@@ -100,8 +87,8 @@ else()
   endif()
   if(NOT _vector_version_output MATCHES "vector ${VECTOR_VERSION} ")
     message(FATAL_ERROR
-      "vector_vendor: downloaded Vector reports version '${_vector_version_output}', "
-      "expected ${VECTOR_VERSION}")
+      "vector_vendor: checked-in Vector binary reports version "
+      "'${_vector_version_output}', expected ${VECTOR_VERSION}")
   endif()
 endif()
 

vector_vendor/package.xml

@@ -9,7 +9,7 @@ SPDX-License-Identifier: MPL-2.0
 <package format="3">
     <name>vector_vendor</name>
     <version>0.1.0</version>
-    <description>Vendor package that downloads a checksum-pinned prebuilt Vector binary at build time (ADR-0002), or vendors a system-installed Vector via the vector_path override.</description>
+    <description>Vendor package that installs a checksum-pinned prebuilt Vector binary checked into this package's git history (ADR-0002), or vendors a system-installed Vector via the vector_path override.</description>
     <maintainer email="d.bensoussan@proton.me">David Bensoussan</maintainer>
     <license>MPL-2.0</license>