Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions src/vector/header.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,30 @@
//! Format version for the per-segment `.vec` file.
//! Format version for the per-segment vector files (`.vec` and `.centroids`).
//!
//! A fixed 4-byte header (a `u32` version) is prepended to every `.vec` file,
//! ahead of the [`CompositeFile`](crate::directory::CompositeFile) body. The
//! version is the wire-layout *generation* — bump it when the framing changes
//! incompatibly. It is orthogonal to the
//! A fixed 4-byte header (a `u32` version) is prepended to every file, ahead of
//! the [`CompositeFile`](crate::directory::CompositeFile) body. The version is
//! the wire-layout *generation* — bump it when the framing changes incompatibly.
//!
//! For `.vec`, the version is orthogonal to the
//! [`IdMap`](super::flat::id_map) variant, which selects the storage *mode*
//! (flat vs IVF) within a generation.
//! (flat vs IVF) within a generation. For `.centroids`, it versions the IVF
//! routing composite (centroids, cluster offsets, optional graph).

use std::io::{self, Read, Write};

use common::{BinarySerializable, HasLen};

use crate::directory::FileSlice;

/// Length of the `.vec` header in bytes (a single `u32`).
/// Length of the version header in bytes (a single `u32`).
pub(crate) const HEADER_LEN: usize = 4;

/// On-disk format version of the `.vec` file.
/// On-disk format version of a vector segment file (`.vec` or `.centroids`).
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum VectorFileVersion {
V1 = 1,
}

/// Version stamped into newly written `.vec` files.
/// Version stamped into newly written vector files.
pub(crate) const CURRENT: VectorFileVersion = VectorFileVersion::V1;

impl BinarySerializable for VectorFileVersion {
Expand All @@ -35,7 +37,7 @@ impl BinarySerializable for VectorFileVersion {
1 => Ok(VectorFileVersion::V1),
other => Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("unsupported .vec format version: {other}"),
format!("unsupported vector file format version: {other}"),
)),
}
}
Expand All @@ -55,7 +57,7 @@ pub(crate) fn read_header(file: &FileSlice) -> io::Result<(VectorFileVersion, Fi
if file.len() < HEADER_LEN {
return Err(io::Error::new(
io::ErrorKind::UnexpectedEof,
"`.vec` file is smaller than its header",
"vector file is smaller than its header",
));
}
let header_bytes = file.slice_to(HEADER_LEN).read_bytes()?;
Expand Down
4 changes: 2 additions & 2 deletions src/vector/index_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,11 +112,11 @@ impl VectorIndexReader {
};
let id_map = IdMap::open(id_map_slice, segment_reader.max_doc())?;

// TODO: Add a version header to the centroids file
let centroid_slots =
match segment_reader.open_read(SegmentComponent::Custom(CENTROIDS_EXT.to_string())) {
Ok(file) => {
let composite = CompositeFile::open(&file)?;
let (_version, body) = read_header(&file)?;
let composite = CompositeFile::open(&body)?;
match (
composite.open_read_with_idx(field, 0),
composite.open_read_with_idx(field, 1),
Expand Down
7 changes: 4 additions & 3 deletions src/vector/ivf/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
//! serializers the merge calls and the [`IvfIndex::open`] that parses them
//! back sit side by side.
//!
//! Written per field, only for IVF segments (⟺ the field's `.vec` `IdMap` is
//! `Explicit`). A [`CompositeFile`](crate::directory::CompositeFile) with
//! three slots per field:
//! The on-disk file is a 4-byte format-version stamp (see `vector::header`)
//! followed by a [`CompositeFile`](crate::directory::CompositeFile). Written
//! per field, only for IVF segments (⟺ the field's `.vec` `IdMap` is
//! `Explicit`). The composite has three slots per field:
//!
//! ```text
//! [0] num_centroids (u32) + num_docs (u32) + centroid_bytes (N · stride)
Expand Down
4 changes: 3 additions & 1 deletion src/vector/ivf/plugin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@
/// trained `centroids` (flat, `dim`-strided) with search beam `ef`. Borrows
/// the flat centroid arena; after the assign loop the same graph is
/// persisted as the `.centroids` slot [2] routing graph.
fn build_centroid_graph<'a>(

Check warning on line 109 in src/vector/ivf/plugin.rs

View workflow job for this annotation

GitHub Actions / clippy

the following explicit lifetimes could be elided: 'a

warning: the following explicit lifetimes could be elided: 'a --> src/vector/ivf/plugin.rs:109:25 | 109 | fn build_centroid_graph<'a>( | ^^ 110 | metric: Metric, 111 | centroids: &'a [f32], | ^^ ... 114 | ) -> crate::Result<RelativeNeighborhoodGraph<&'a [f32]>> { | ^^ | = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.97.0/index.html#needless_lifetimes = note: `#[warn(clippy::needless_lifetimes)]` on by default help: elide the lifetimes | 109 ~ fn build_centroid_graph( 110 | metric: Metric, 111 ~ centroids: &[f32], 112 | dim: usize, 113 | ef: usize, 114 ~ ) -> crate::Result<RelativeNeighborhoodGraph<&[f32]>> { |

Check warning on line 109 in src/vector/ivf/plugin.rs

View workflow job for this annotation

GitHub Actions / clippy

the following explicit lifetimes could be elided: 'a

warning: the following explicit lifetimes could be elided: 'a --> src/vector/ivf/plugin.rs:109:25 | 109 | fn build_centroid_graph<'a>( | ^^ 110 | metric: Metric, 111 | centroids: &'a [f32], | ^^ ... 114 | ) -> crate::Result<RelativeNeighborhoodGraph<&'a [f32]>> { | ^^ | = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.97.0/index.html#needless_lifetimes = note: `#[warn(clippy::needless_lifetimes)]` on by default help: elide the lifetimes | 109 ~ fn build_centroid_graph( 110 | metric: Metric, 111 ~ centroids: &[f32], 112 | dim: usize, 113 | ef: usize, 114 ~ ) -> crate::Result<RelativeNeighborhoodGraph<&[f32]>> { |
metric: Metric,
centroids: &'a [f32],
dim: usize,
Expand Down Expand Up @@ -217,7 +217,9 @@
let mut vec_file = directory.open_write(&vec_path)?;
write_header(&mut vec_file)?;
let mut vec_write = CompositeWrite::wrap(vec_file);
let mut centroids_write = CompositeWrite::wrap(directory.open_write(&centroids_path)?);
let mut centroids_file = directory.open_write(&centroids_path)?;
write_header(&mut centroids_file)?;
let mut centroids_write = CompositeWrite::wrap(centroids_file);

for (field, entry) in ctx.schema.fields() {
let opts = match entry.field_type() {
Expand Down
51 changes: 50 additions & 1 deletion src/vector/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,54 @@
Ok(())
}

/// Both vector segment files must stamp the current format-generation header
/// ahead of their composite body, so future layout changes can be gated.
#[test]
fn vector_files_stamp_format_version_header() -> crate::Result<()> {
use crate::directory::CompositeFile;
use crate::index::SegmentComponent;
use crate::vector::header::{read_header, VectorFileVersion};
use crate::vector::ivf::CENTROIDS_EXT;
use crate::vector::VEC_EXT;

for format in [VectorStorageFormat::Flat, VectorStorageFormat::Ivf] {
let index = TestVectorIndex::builder(VectorDType::F32)
.vector_storage_format(format)
.build()?;
let searcher = index.index.reader()?.searcher();
assert!(!searcher.segment_readers().is_empty());

for segment_reader in searcher.segment_readers() {
let vec_file =
segment_reader.open_read(SegmentComponent::Custom(VEC_EXT.to_string()))?;
let (version, body) = read_header(&vec_file)?;
assert_eq!(version, VectorFileVersion::V1);
// Body must be a valid composite — proves the stamp sits in front
// of the framing, not inside a slot.
CompositeFile::open(&body)?;

match format {
VectorStorageFormat::Flat => {
assert!(
segment_reader
.open_read(SegmentComponent::Custom(CENTROIDS_EXT.to_string()))
.is_err(),
"flat segments must not write `.centroids`"
);
}
VectorStorageFormat::Ivf => {
let centroids_file = segment_reader
.open_read(SegmentComponent::Custom(CENTROIDS_EXT.to_string()))?;
let (version, body) = read_header(&centroids_file)?;
assert_eq!(version, VectorFileVersion::V1);
CompositeFile::open(&body)?;
}
}
}
}
Ok(())
}

#[test]
fn fixture_vectors_round_trip_from_readers() -> crate::Result<()> {
let mut expected = grid2d::vectors(NUM_DOCS);
Expand Down Expand Up @@ -435,7 +483,8 @@
for segment_reader in searcher.segment_readers() {
let centroids_file =
segment_reader.open_read(SegmentComponent::Custom(CENTROIDS_EXT.to_string()))?;
let composite = CompositeFile::open(&centroids_file)?;
let (_version, body) = super::header::read_header(&centroids_file)?;
let composite = CompositeFile::open(&body)?;
let graph_bytes = composite
.open_read_with_idx(index.embedding_field(), 2)
.expect("IVF merge should write the centroid graph slot")
Expand Down Expand Up @@ -748,7 +797,7 @@
}

fn points_per_cluster(ndocs: usize, num_clusters: usize) -> usize {
(ndocs + num_clusters - 1) / num_clusters

Check warning on line 800 in src/vector/tests.rs

View workflow job for this annotation

GitHub Actions / clippy

manually reimplementing `div_ceil`

warning: manually reimplementing `div_ceil` --> src/vector/tests.rs:800:9 | 800 | (ndocs + num_clusters - 1) / num_clusters | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: consider using `.div_ceil()`: `ndocs.div_ceil(num_clusters)` | = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.97.0/index.html#manual_div_ceil = note: `#[warn(clippy::manual_div_ceil)]` on by default
}

fn grid(origin: [f32; DIM], rows: usize, cols: usize, gap: f32) -> Vec<[f32; DIM]> {
Expand Down
Loading