Merge branch 'master' into feature/memory-enrichment-table-cuckoo-filter

Author: esensar

.github/CODEOWNERS

@@ -3,6 +3,8 @@
 .github/workflows/regression.yml @vectordotdev/vector @vectordotdev/single-machine-performance
 regression/config.yaml @vectordotdev/vector @vectordotdev/single-machine-performance
 
+tests/antithesis/ @vectordotdev/vector @vectordotdev/single-machine-performance
+
 website/ @vectordotdev/vector
 website/js @vectordotdev/vector @vectordotdev/vector-website
 website/layouts @vectordotdev/vector @vectordotdev/vector-website

.gitignore

@@ -62,6 +62,7 @@ volumes/
 # Environment variable files (can contain secrets)
 *.env
 .env.*
+!tests/antithesis/scenarios/*/launch.env
 
 # LLM tools
 copilot-instructions.md
@@ -76,3 +77,9 @@ local/
 .claude/skill-reports/
 # Local Claude Code settings (machine-specific permissions)
 .claude/settings.local.json
+
+# antithesis scratchbook (private knowledge store)
+/tests/antithesis/scratchbook/
+
+# stray rlib build artifacts
+*.rlib

Cargo.lock

@@ -132,14 +132,18 @@ members = [
   "lib/vector-vrl/metrics",
   "lib/vector-vrl/tests",
   "lib/vector-vrl/web-playground",
+  "tests/antithesis/harness",
   "vdev",
 ]
 
 [workspace.dependencies]
+antithesis-instrumentation = { version = "0.1", default-features = false, features = [] }
+antithesis_sdk = { version = "0.2", default-features = false, features = [] }
 anyhow = { version = "1.0.102", default-features = false, features = ["std"] }
 arc-swap = { version = "1.8.2", default-features = false }
 async-stream = { version = "0.3.6", default-features = false }
 async-trait = { version = "0.1.89", default-features = false }
+axum = { version = "0.6.20", default-features = false }
 base64 = { version = "0.22.1", default-features = false }
 bytes = { version = "1.11.1", default-features = false, features = ["serde"] }
 cfg-if = { version = "1.0.4", default-features = false }
@@ -213,6 +217,7 @@ tracing = { version = "0.1.44", default-features = false }
 tracing-subscriber = { version = "0.3.22", default-features = false, features = ["fmt"] }
 url = { version = "2.5.4", default-features = false, features = ["serde"] }
 uuid = { version = "1.22.0", features = ["v4", "v7", "serde", "fast-rng"] }
+vector-buffers = { path = "lib/vector-buffers" }
 vector-config = { path = "lib/vector-config" }
 vector-config-common = { path = "lib/vector-config-common" }
 vector-config-macros = { path = "lib/vector-config-macros" }
@@ -482,6 +487,8 @@ tikv-jemallocator = { version = "0.7.0", default-features = false, features = ["
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.18.0", default-features = false }
+# antithesis-instrumentation links a native .a via ELF .init_array -- Linux only.
+antithesis-instrumentation = { workspace = true, optional = true }
 
 [build-dependencies]
 prost-build = { workspace = true, optional = true }
@@ -530,6 +537,8 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3
 # table below). Local dev builds skip it so they don't need `libsasl2-dev` / cyrus-sasl
 # installed, and so rdkafka's build script doesn't link GSSAPI.
 default = ["enable-unix"]
+antithesis-scenario-memory = ["dep:antithesis-instrumentation"]
+antithesis-scenario-disk = ["dep:antithesis-instrumentation", "vector-lib/antithesis-disk-asserts"]
 # Default features for `cargo docs`. We're not using `gssapi` which would require installing libsasl2 in our doc environment.
 docs = ["enable-unix"]
 # Default features for *-unknown-linux-* which make use of `cmake` for dependencies

Cargo.toml

@@ -18,6 +18,8 @@ anstyle,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle Author
 anstyle-parse,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-parse Authors
 anstyle-query,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-query Authors
 anstyle-wincon,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The anstyle-wincon Authors
+antithesis-instrumentation,https://github.com/antithesishq/antithesis-instrumentation-rust,MIT,The antithesis-instrumentation Authors
+antithesis_sdk,https://github.com/antithesishq/antithesis-sdk-rust,MIT,The antithesis_sdk Authors
 anyhow,https://github.com/dtolnay/anyhow,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 apache-avro,https://github.com/apache/avro-rs,Apache-2.0,The apache-avro Authors
 arbitrary,https://github.com/rust-fuzz/arbitrary,MIT OR Apache-2.0,"The Rust-Fuzz Project Developers, Nick Fitzgerald <fitzgen@gmail.com>, Manish Goregaokar <manishsmail@gmail.com>, Simonas Kazlauskas <arbitrary@kazlauskas.me>, Brian L. Troutwine <brian@troutwine.us>, Corey Farwell <coreyf@rwell.org>"
@@ -433,13 +435,16 @@ lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex
 lexical-write-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
 lexical-write-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
 libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
+libloading,https://github.com/nagisa/rust_libloading,ISC,Simonas Kazlauskas <libloading@kazlauskas.me>
 libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio <jorge@japaric.io>
 libredox,https://gitlab.redox-os.org/redox-os/libredox,MIT,4lDO2 <4lDO2@protonmail.com>
 libsqlite3-sys,https://github.com/rusqlite/rusqlite,MIT,The rusqlite developers
 libz-sys,https://github.com/rust-lang/libz-sys,MIT OR Apache-2.0,"Alex Crichton <alex@alexcrichton.com>, Josh Triplett <josh@joshtriplett.org>, Sebastian Thiel <sebastian.thiel@icloud.com>"
 line-clipping,https://github.com/joshka/line-clipping,MIT OR Apache-2.0,Josh McKinney
 linked-hash-map,https://github.com/contain-rs/linked-hash-map,MIT OR Apache-2.0,"Stepan Koltsov <stepan.koltsov@gmail.com>, Andrew Paseltiner <apaseltiner@gmail.com>"
 linked_hash_set,https://github.com/alexheretic/linked-hash-set,Apache-2.0,Alex Butler <alexheretic@gmail.com>
+linkme,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
+linkme-impl,https://github.com/dtolnay/linkme,MIT OR Apache-2.0,David Tolnay <dtolnay@gmail.com>
 linux-raw-sys,https://github.com/sunfishcode/linux-raw-sys,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Dan Gohman <dev@sunfishcode.online>
 listenfd,https://github.com/mitsuhiko/listenfd,Apache-2.0,Armin Ronacher <armin.ronacher@active-4.com>
 litemap,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers

LICENSE-3rdparty.csv

@@ -0,0 +1,6 @@
+Add support for optionally applying rate limiting to the `internal_logs` source controlled by the
+`--internal-logs-source-rate-limit` CLI option and `VECTOR_INTERNAL_LOGS_SOURCE_RATE_LIMIT`
+environment variable. This provides the same rate limiting functionality as was available before
+version 0.51.1 but with a rate limit window separate from the console one.
+
+authors: bruceg

changelog.d/internal_logs_broadcast_rate_limit.enhancement.md

@@ -264,7 +264,8 @@ to detect common problems.
 
 ### Disabling internal log rate limiting
 
-Vector rate limits its own internal logs by default (10-second windows). During development, you may want to see all log occurrences.
+Vector rate limits the console output of internal logs by default (10-second windows). During
+development, you may want to see all log occurrences.
 
 **Globally** (CLI flag or environment variable):
 
@@ -284,6 +285,10 @@ warn!(message = "Error occurred.", %error, internal_log_rate_limit = false);
 info!(message = "Processing batch.", batch_size, internal_log_rate_secs = 1);
 ```
 
+Note: The `internal_logs` source is _not_ rate limited by default. To enable rate limiting on all
+such sources, set the `--internal-logs-source-rate-limit` CLI flag or
+`VECTOR_INTERNAL_LOGS_SOURCE_RATE_LIMIT` environment variable to an integer number of seconds.
+
 ## Testing
 
 Testing is very important since Vector's primary design principle is reliability.

docs/DEVELOPING.md

@@ -13,6 +13,10 @@
 //! - **3rd+ occurrences**: Silent until window expires
 //! - **After window**: Emits a summary of suppressed count, then next event normally
 //!
+//! Note: the suppressed-count summary and the resumption of normal emission are both
+//! triggered by the *next arriving event* after the window has elapsed, not by the
+//! window expiry itself. If the event stops firing, no summary is ever emitted.
+//!
 //! # Rate limit grouping
 //!
 //! Events are rate limited independently based on a combination of:
@@ -163,6 +167,7 @@ where
     /// - 1st occurrence: Emitted normally
     /// - 2nd occurrence: Shows "suppressing" warning
     /// - 3rd+ occurrences: Silent until window expires
+    /// - After window: Summary and next event emitted on next arrival (see module-level note)
     pub fn with_default_limit(mut self, internal_log_rate_limit: u64) -> Self {
         self.internal_log_rate_limit = internal_log_rate_limit;
         self

lib/tracing-limit/src/lib.rs

@@ -34,6 +34,12 @@ vector-config = { path = "../vector-config", default-features = false }
 vector-common = { path = "../vector-common", default-features = false, features = ["byte_size_of"] }
 dashmap.workspace = true
 ordered-float.workspace = true
+antithesis_sdk = { workspace = true, optional = true, features = ["full"] }
+serde_json = { workspace = true, optional = true }
+
+[features]
+antithesis-disk-asserts = ["dep:antithesis_sdk", "dep:serde_json"]
+test = []
 
 [dev-dependencies]
 clap.workspace = true
@@ -45,6 +51,7 @@ metrics-util = { workspace = true, features = ["debugging"] }
 proptest.workspace = true
 quickcheck.workspace = true
 rand.workspace = true
+serde_json.workspace = true
 serde_yaml.workspace = true
 temp-dir = "0.1.16"
 tokio-test.workspace = true

lib/vector-buffers/Cargo.toml

@@ -32,6 +32,11 @@ pub mod topology;
 
 pub(crate) mod variants;
 
+/// `disk_v2`'s write-buffer size, re-exported under `test` so the harness can
+/// size payloads against the real value instead of hardcoding it.
+#[cfg(feature = "test")]
+pub use variants::disk_v2::common::DEFAULT_WRITE_BUFFER_SIZE as WRITE_BUFFER_SIZE_V2;
+
 use std::fmt::Debug;
 
 #[cfg(test)]