Skip to content

Commit 1b72428

Browse files
committed
Share pre-computed block between file write children
Previously we constructed a separate block cache for each write child in `file_gen`. This meant as we got more duplicate file writes we increased the memory by the prebuild cache size. This lead to situations where lading would OOM if the number of write children were high enough. REF SMP-558 Signed-off-by: Brian L. Troutwine <brian.troutwine@datadoghq.com>
1 parent 5bc172f commit 1b72428

2 files changed

Lines changed: 10 additions & 5 deletions

File tree

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
66

77
## [Unreleased]
88

9+
## [0.15.3-rc1]
10+
### Changed
11+
- `file_gen` now shares its pre-computed block between write children, reducing
12+
memory consumption.
13+
914
## [0.15.2]
1015
### Fixed
1116
- Disallow the creation of DogStatsD metrics with no values

src/generator/file_gen.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -156,18 +156,18 @@ impl FileGen {
156156

157157
let mut handles = Vec::new();
158158
let file_index = Arc::new(AtomicU32::new(0));
159+
let block_cache = construct_block_cache(&mut rng, &config.variant, &block_chunks, &labels);
160+
let block_cache = Arc::new(block_cache);
161+
159162
for _ in 0..config.duplicates {
160163
let throttle = Throttle::new_with_config(config.throttle, bytes_per_second);
161164

162-
let block_cache =
163-
construct_block_cache(&mut rng, &config.variant, &block_chunks, &labels);
164-
165165
let child = Child {
166166
path_template: config.path_template.clone(),
167167
maximum_bytes_per_file,
168168
bytes_per_second,
169169
throttle,
170-
block_cache,
170+
block_cache: Arc::clone(&block_cache),
171171
file_index: Arc::clone(&file_index),
172172
rotate: config.rotate,
173173
shutdown: shutdown.clone(),
@@ -210,7 +210,7 @@ struct Child {
210210
maximum_bytes_per_file: NonZeroU32,
211211
bytes_per_second: NonZeroU32,
212212
throttle: Throttle,
213-
block_cache: Vec<Block>,
213+
block_cache: Arc<Vec<Block>>,
214214
rotate: bool,
215215
file_index: Arc<AtomicU32>,
216216
shutdown: Shutdown,

0 commit comments

Comments
 (0)