chore(ci): pin Rust MSRV to 1.94 (#2035)

## Summary

Pin Saluki's Rust MSRV to 1.94 and align the Rust toolchain pins used by local development, Cargo metadata, CI images, Docker build defaults, and nightly-only tooling.

## Changes

- Set `workspace.package.rust-version = "1.94"` in the root `Cargo.toml`.
- Keep `rust-version = "1.94"` in the standalone `lib/saluki-io/fuzz/Cargo.toml`, which cannot inherit from the root workspace.
- Pin `rust-toolchain.toml` to `1.94.0`.
- Update Rust defaults in CI/build Dockerfiles from `1.93.0`/`stable` to `1.94.0`.
- Pin nightly-only tooling to `nightly-2026-01-18`, which reports `rustc 1.94.0-nightly`.
- Replace unqualified `cargo +nightly` commands in Makefile, GitLab fuzzing, docs, and the fuzz infrastructure script with the pinned nightly version.
- Replace uses of unstable `Atomic*::try_update` with stable `fetch_update` so the workspace builds on Rust 1.94.
- Pass an explicit cwd to `cargo sort --workspace` in Makefile fmt targets so formatting checks work from linked worktrees.
- Make the `host_exec_times_out` unit test use a Windows-native sleep command so it exercises timeout behavior on Windows CI.
- Add retrying cleanup for macOS Datadog Agent DMG mounts in both provisioning and CI before/after hooks for persistent runners.
- Bump `crossbeam-epoch` in `Cargo.lock` to `0.9.20` to resolve `RUSTSEC-2026-0204` reported by `cargo-deny` during pre-commit.

## Test plan

- `cargo +1.94.0 nextest run -p panoramic actions::target_exec::tests::host_exec_times_out`
- `cargo +1.94.0 check --workspace --all-targets`
- `make check-fmt`
- `make generate-api-docs`
- `make check-deny`
- `make check-docs`
- `python3 -m py_compile ci/tooling/fuzz_infra.py`
- `bash -n ci/tooling/cleanup-macos-dda-mounts.sh`
- `make -n provision-macos-test-env`
- `git diff --check`
- Pre-commit hook completed successfully, including fmt, clippy, license check, cargo-deny, Vale, and API docs check.


Co-authored-by: travis.thieman <travis.thieman@datadoghq.com>

Author: thieman

.github/workflows/docs.yml

@@ -34,11 +34,11 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y protobuf-compiler
       # `make generate-api-docs` installs and pins the nightly toolchain it needs (see
-      # RUST_NIGHTLY_VERSION in the Makefile); this step just provides rustup/cargo.
-      - name: Set up Rust
+      # RUST_NIGHTLY_VERSION in the Makefile); this step just provides Rust 1.94 and rustup/cargo.
+      - name: Set up Rust 1.94
         uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17
         with:
-          toolchain: stable
+          toolchain: 1.94.0
           cache: false
           rustflags: ""
       - name: Generate API documentation

.gitlab/e2e.yml

@@ -159,7 +159,9 @@ test-integration:
     #     $CI_PROJECT_DIR/target/<profile>/, holds UDP 58125 / TCP 5100–5102 etc. across runs).
     #     Both profiles are swept because BUILD_PROFILE flows from .gitlab-ci.yml workflow
     #     rules: `release` on dev pipelines, `optimized-release` on tagged release pipelines.
+    #   - stale Datadog Agent DMG mounts from interrupted installs; these runners persist across jobs.
     - sudo pkill -9 -f /tmp/saluki-dda/ || true
+    - bash ci/tooling/cleanup-macos-dda-mounts.sh || true
     - sudo pkill -9 -f /target/release/agent-data-plane || true
     - sudo pkill -9 -f /target/optimized-release/agent-data-plane || true
   script:
@@ -177,6 +179,7 @@ test-integration:
     - ps -axo pid,ppid,user,command > integration-logs/host-diag/ps.txt 2>&1 || true
     - cp /tmp/saluki-agent-bootstrap.log integration-logs/host-diag/saluki-agent-bootstrap.log 2>/dev/null || true
     - ls -la /tmp/saluki-dda/datadog-agent/etc/ > integration-logs/host-diag/sandbox-etc.txt 2>&1 || true
+    - bash ci/tooling/cleanup-macos-dda-mounts.sh || true
 
 test-integration-macos-arm64:
   extends:

.gitlab/fuzz.yml

@@ -10,8 +10,10 @@ fuzz:infra:
     - if: '$CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push"'
     - when: manual
       allow_failure: true
+  variables:
+    RUST_NIGHTLY_VERSION: nightly-2026-01-18
   before_script:
-    - cargo +nightly install cargo-fuzz
+    - cargo +${RUST_NIGHTLY_VERSION} install cargo-fuzz
     # Install Python dependencies for fuzz_infra.py
     - apt-get update && apt-get install -y python3 python3-pip
     - pip3 install requests toml # parsing of cargo toml files

Cargo.toml

@@ -44,6 +44,7 @@ resolver = "2"
 
 [workspace.package]
 edition = "2021"
+rust-version = "1.94"
 license = "Apache-2.0"
 repository = "https://github.com/DataDog/saluki"
 

Makefile

@@ -53,7 +53,7 @@ export GO_APP_IMAGE ?= ubuntu:24.04
 # Pinned nightly toolchain shared by the Miri tests and API-doc generation, both of which rely on
 # nightly-only features. Keeping it in one variable ensures the two stay in lockstep; bump here to
 # move both at once.
-export RUST_NIGHTLY_VERSION ?= nightly-2026-07-05
+export RUST_NIGHTLY_VERSION ?= nightly-2026-01-18
 
 # Tool configuration.
 export AUTOINSTALL ?= true
@@ -525,12 +525,12 @@ check-deny: ## Check all crate dependencies for outstanding advisories or usage
 	@cargo deny check --hide-inclusion-graph --show-stats
 
 .PHONY: check-fmt
-check-fmt: check-rust-build-tools cargo-install-cargo-sort
+check-fmt: check-rust-build-tools ensure-rust-nightly cargo-install-cargo-sort
 check-fmt: ## Check that all Rust source files are formatted properly
 	@echo "[*] Checking Rust source code formatting..."
-	@cargo +nightly fmt -- --check
+	@cargo +$(RUST_NIGHTLY_VERSION) fmt -- --check
 	@echo "[*] Checking Cargo.toml formatting..."
-	@cargo sort --workspace --check >/dev/null
+	@cargo sort --workspace . --check >/dev/null
 
 .PHONY: check-licenses
 check-licenses: check-rust-build-tools cargo-install-dd-rust-license-tool
@@ -697,12 +697,24 @@ provision-macos-test-env: ## Installs the pinned Datadog Agent ($(MACOS_TEST_AGE
 			curl -fL "$(MACOS_TEST_AGENT_DMG_URL)" -o "$$DMG_PATH"; \
 		fi; \
 		MOUNT_DIR=$$(mktemp -d /tmp/saluki-dda-mount-XXXXXX); \
-		hdiutil attach "$$DMG_PATH" -mountpoint "$$MOUNT_DIR" -nobrowse >/dev/null; \
+		cleanup_mount() { bash "$(CURDIR)/ci/tooling/cleanup-macos-dda-mounts.sh" "$$MOUNT_DIR"; }; \
+		trap cleanup_mount EXIT; \
+		for attempt in 1 2 3; do \
+			if hdiutil attach "$$DMG_PATH" -mountpoint "$$MOUNT_DIR" -nobrowse >/dev/null; then \
+				break; \
+			fi; \
+			if [ "$$attempt" = "3" ]; then \
+				echo "ERROR: failed to attach $$DMG_PATH after $$attempt attempts" >&2; \
+				exit 1; \
+			fi; \
+			echo "[*] Failed to attach $$DMG_PATH on attempt $$attempt; retrying..." >&2; \
+			sleep $$((attempt * 2)); \
+		done; \
 		PKG=$$(find "$$MOUNT_DIR" -name '*.pkg' | head -1); \
 		EXPAND_DIR=$$(mktemp -d /tmp/saluki-dda-expand-XXXXXX) && rm -rf "$$EXPAND_DIR"; \
 		pkgutil --expand-full "$$PKG" "$$EXPAND_DIR" >/dev/null; \
-		hdiutil detach "$$MOUNT_DIR" >/dev/null; \
-		rmdir "$$MOUNT_DIR" 2>/dev/null || true; \
+		cleanup_mount; \
+		trap - EXIT; \
 		PAYLOAD_DIR=$$(find "$$EXPAND_DIR" -type d -name Payload | head -1); \
 		if [ -z "$$PAYLOAD_DIR" ] || [ ! -x "$$PAYLOAD_DIR/bin/agent/agent" ]; then \
 			echo "ERROR: pkg payload did not contain bin/agent/agent. Expanded layout:" >&2; \
@@ -755,7 +767,7 @@ ifeq ($(shell command -v rustup >/dev/null || echo not-found), not-found)
 	$(error "Rustup must be present to install the nightly toolchain: https://www.rust-lang.org/tools/install")
 endif
 	@echo "[*] Installing/updating nightly Rust ($(RUST_NIGHTLY_VERSION))..."
-	@rustup toolchain install $(RUST_NIGHTLY_VERSION) --profile minimal
+	@rustup toolchain install $(RUST_NIGHTLY_VERSION) --profile minimal --component rustfmt
 
 .PHONY: ensure-rust-miri
 ensure-rust-miri: ensure-rust-nightly
@@ -949,14 +961,14 @@ clean-kind: check-kind-tools ## Cleans up orphaned panoramic namespaces in the k
 clean-correctness: clean-airlock clean-kind ## Cleans up all orphaned correctness test resources (Docker + kind)
 
 .PHONY: fmt
-fmt: check-rust-build-tools cargo-install-cargo-autoinherit cargo-install-cargo-sort
+fmt: check-rust-build-tools ensure-rust-nightly cargo-install-cargo-autoinherit cargo-install-cargo-sort
 fmt: ## Format Rust source code
 	@echo "[*] Formatting Rust source code..."
-	@cargo +nightly fmt
+	@cargo +$(RUST_NIGHTLY_VERSION) fmt
 	@echo "[*] Ensuring workspace dependencies are autoinherited..."
 	@cargo autoinherit 2>/dev/null
 	@echo "[*] Formatting Cargo.toml files..."
-	@cargo sort --workspace >/dev/null
+	@cargo sort --workspace . >/dev/null
 
 .PHONY: sync-licenses
 sync-licenses: check-rust-build-tools cargo-install-dd-rust-license-tool

bin/correctness/panoramic/src/actions/target_exec.rs

@@ -236,17 +236,38 @@ mod tests {
         assert!(err.to_string().contains("fail-message"));
     }
 
+    fn slow_host_command() -> Vec<String> {
+        #[cfg(windows)]
+        {
+            vec![
+                "powershell".to_string(),
+                "-NoProfile".to_string(),
+                "-NonInteractive".to_string(),
+                "-Command".to_string(),
+                "Start-Sleep -Seconds 5".to_string(),
+            ]
+        }
+
+        #[cfg(not(windows))]
+        {
+            vec!["sh".to_string(), "-c".to_string(), "sleep 5".to_string()]
+        }
+    }
+
     #[tokio::test]
     async fn host_exec_times_out() {
         let err = exec_on_host_with_timeout(
-            &["sh".to_string(), "-c".to_string(), "sleep 5".to_string()],
+            &slow_host_command(),
             std::time::Duration::from_millis(50),
             &tokio_util::sync::CancellationToken::new(),
             &tokio_util::sync::CancellationToken::new(),
         )
         .await
         .expect_err("host command should time out");
 
-        assert!(err.to_string().contains("Timed out running host command"));
+        assert!(
+            err.to_string().contains("Timed out running host command"),
+            "unexpected error: {err}"
+        );
     }
 }

ci/images/definitions/build/Dockerfile

@@ -14,7 +14,7 @@ COPY bin/agent-data-plane/Cargo.toml /tmp/bin/agent-data-plane/Cargo.toml
 
 FROM ${BASE_IMAGE}
 
-ARG RUST_VERSION=1.93.0
+ARG RUST_VERSION=1.94.0
 ARG TARGETARCH
 
 # Base image defaults to the unprivileged `dog` user; switch to root for all install steps.

ci/images/definitions/smp/Dockerfile

@@ -28,7 +28,7 @@ RUN chmod +x /install-buildcache.sh && /install-buildcache.sh
 # Final stage: runtime-only dependencies. Build toolchain is not present here.
 FROM registry.ddbuild.io/docker:24.0.4-jammy
 
-ARG RUST_VERSION=1.93.0
+ARG RUST_VERSION=1.94.0
 
 ENV DEBIAN_FRONTEND=noninteractive \
     TZ=Etc/UTC

ci/tooling/cleanup-macos-dda-mounts.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Cleans up temporary Datadog Agent DMG mount points left by macOS integration-test setup.
+set -euo pipefail
+
+retry_detach() {
+  local mount_dir="$1"
+
+  if [[ ! -d "$mount_dir" ]]; then
+    return 0
+  fi
+
+  if ! mount | grep -F " on ${mount_dir} " >/dev/null 2>&1 && ! hdiutil info | grep -F "$mount_dir" >/dev/null 2>&1; then
+    rmdir "$mount_dir" 2>/dev/null || true
+    return 0
+  fi
+
+  local attempt
+  for attempt in 1 2 3; do
+    if hdiutil detach "$mount_dir" >/dev/null 2>&1; then
+      rmdir "$mount_dir" 2>/dev/null || true
+      return 0
+    fi
+
+    echo "[*] Failed to detach ${mount_dir} on attempt ${attempt}; retrying..." >&2
+    sleep $((attempt * 2))
+  done
+
+  echo "[*] Forcing detach of ${mount_dir} after retries failed..." >&2
+  hdiutil detach -force "$mount_dir" >/dev/null 2>&1 || true
+  rmdir "$mount_dir" 2>/dev/null || true
+}
+
+if [[ "$#" -gt 0 ]]; then
+  for mount_dir in "$@"; do
+    retry_detach "$mount_dir"
+  done
+else
+  shopt -s nullglob
+  for mount_dir in /tmp/saluki-dda-mount-*; do
+    retry_detach "$mount_dir"
+  done
+fi

ci/tooling/fuzz_infra.py

@@ -14,6 +14,7 @@
 import toml
 
 DEFAULT_FUZZING_SLACK_CHANNEL = "fuzzing-ops"
+DEFAULT_RUST_NIGHTLY_VERSION = "nightly-2026-01-18"
 
 # Lets reuse the token for all requests to avoid issues with rate limiting.
 # The process should be short lived enough that the token should be valid for the duration (it's only the uploading step)
@@ -143,8 +144,12 @@ def upload_fuzz(
     print(response_json)
 
 
+def get_rust_nightly_version() -> str:
+    return os.getenv("RUST_NIGHTLY_VERSION", DEFAULT_RUST_NIGHTLY_VERSION)
+
+
 def search_fuzz_tests(directory) -> list[str]:
-    fuzz_list_cmd = ["cargo", "+nightly", "fuzz", "list"]
+    fuzz_list_cmd = ["cargo", f"+{get_rust_nightly_version()}", "fuzz", "list"]
     process = Popen(fuzz_list_cmd, cwd=directory, stdout=PIPE, stderr=PIPE)
     stdout, stderr = process.communicate()
 
@@ -162,7 +167,7 @@ def search_fuzz_tests(directory) -> list[str]:
 
 
 def build_fuzz(directory, fuzz_test) -> bool:
-    build_cmd = ["cargo", "+nightly", "fuzz", "build", fuzz_test]
+    build_cmd = ["cargo", f"+{get_rust_nightly_version()}", "fuzz", "build", fuzz_test]
     return Popen(build_cmd, cwd=directory).wait() == 0