Skip to content

Commit fdf0a79

Browse files
stuhoodrebasedming
authored andcommitted
Avoid copying into OwnedBytes when opening a fast field column Dictionary. (#55)
When a fast fields string/bytes `Dictionary` is opened, we currently read the entire dictionary from `FileSlice` -> `OwnedBytes`... and then immediately wrap it back into a `FileSlice`. Switching to `Dictionary::open` preserves the `FileSlice`, such that only the portions of the `Dictionary` which are actually accessed are read from disk/buffers. (cherry picked from commit d15966f)
1 parent 8c05e26 commit fdf0a79

3 files changed

Lines changed: 4 additions & 5 deletions

File tree

columnar/src/column/serialize.rs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,7 @@ pub fn open_column_bytes(
120120
);
121121
let (dictionary_bytes, column_bytes) = body.split(dictionary_len as usize);
122122

123-
let dictionary_bytes = dictionary_bytes.read_bytes()?;
124-
let dictionary = Arc::new(Dictionary::from_bytes(dictionary_bytes)?);
123+
let dictionary = Arc::new(Dictionary::open(dictionary_bytes)?);
125124
let term_ord_column = crate::column::open_column_u64::<u64>(column_bytes, format_version)?;
126125
Ok(BytesColumn {
127126
dictionary,

sstable/src/dictionary.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ impl Dictionary<VoidSSTable> {
5555
dictionary_writer.insert(term, &()).unwrap();
5656
}
5757
dictionary_writer.finish().unwrap();
58-
Dictionary::from_bytes(OwnedBytes::new(buffer)).unwrap()
58+
Dictionary::from_bytes_for_tests(OwnedBytes::new(buffer)).unwrap()
5959
}
6060
}
6161

@@ -297,7 +297,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
297297
}
298298

299299
/// Creates a term dictionary from the supplied bytes.
300-
pub fn from_bytes(owned_bytes: OwnedBytes) -> io::Result<Self> {
300+
pub fn from_bytes_for_tests(owned_bytes: OwnedBytes) -> io::Result<Self> {
301301
Dictionary::open(FileSlice::new(Arc::new(owned_bytes)))
302302
}
303303

sstable/src/streamer.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ mod tests {
307307
dict_builder.insert(b"abandon", &3)?;
308308
let buffer = dict_builder.finish()?;
309309
let owned_bytes = OwnedBytes::new(buffer);
310-
Dictionary::from_bytes(owned_bytes)
310+
Dictionary::from_bytes_for_tests(owned_bytes)
311311
}
312312

313313
#[test]

0 commit comments

Comments
 (0)