Skip to content

Commit 070db84

Browse files
achref-souaachref-soua
authored
chore(release): sync develop to main for v0.33.0 (#395)
1 parent dfb60cd commit 070db84

19 files changed

Lines changed: 623 additions & 94 deletions

‎CHANGELOG.md‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,26 @@ for the per-release rationale and Definitions of Done.
1010

1111
## [Unreleased]
1212

13+
## [0.33.0] — Unburdened — 2026-07-04
14+
15+
### Added
16+
17+
- **Streaming, memory-bounded index build — the live wiring** ([ADR-0071](docs/adr/0071-streaming-rebuild-vector-source.md), completing [ADR-0070](docs/adr/0070-streaming-index-build.md) Increment B).
18+
The IVF index rebuild — both the server's off-lock path (ADR-0062) and the
19+
embedded open/`ensure_indexed` path — now streams its vectors from the immutable
20+
on-disk `.vec` segments instead of materialising the whole `n·dim` corpus in a
21+
resident `flat` array (~51 GiB at 100M×128). `Store::capture_vector_source`
22+
reopens the referenced segments' `.vec` `mmap`s as owned, lock-free handles that
23+
a concurrent checkpoint cannot disturb; `scan_collection` captures one for a
24+
dense IVF collection and rebuilds the (usually empty) sparse index from a
25+
payload-only scan, and both rebuild entry points feed `Ivf::build_streaming`.
26+
Peak build memory for the vectors drops from `O(n·dim)` to
27+
`O(sample + nlist·dim + n·m_bytes)`; the corpus stays on disk, read on demand.
28+
No on-disk or wire format change; defaults unchanged. The reference-hardware RSS
29+
measurement (≥ 20M) is deferred to the dedicated box and no number is claimed —
30+
the change lands on correctness tests (a rebuild spanning a reopened sealed
31+
segment and the active buffer) and the code-level elimination of the arena.
32+
1333
## [0.32.0] — Streaming — 2026-07-04
1434

1535
Foundational work toward the single-box **100M** build, plus a cluster-search
@@ -928,7 +948,8 @@ and dynamic, elastic membership with online rebalancing behind a coordinator
928948
SIMD kernels; REST + gRPC; encryption-at-rest by default; TLS via `rustls`; the
929949
TUI MVP; the benchmark harness with first SIFT1M numbers; the Python SDK.
930950

931-
[Unreleased]: https://github.com/achref-soua/quiver/compare/v0.32.0...HEAD
951+
[Unreleased]: https://github.com/achref-soua/quiver/compare/v0.33.0...HEAD
952+
[0.33.0]: https://github.com/achref-soua/quiver/compare/v0.32.0...v0.33.0
932953
[0.32.0]: https://github.com/achref-soua/quiver/compare/v0.31.0...v0.32.0
933954
[0.31.0]: https://github.com/achref-soua/quiver/compare/v0.30.2...v0.31.0
934955
[0.30.2]: https://github.com/achref-soua/quiver/compare/v0.30.1...v0.30.2

‎Cargo.lock‎

Lines changed: 14 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ resolver = "3"
77
members = ["crates/*"]
88

99
[workspace.package]
10-
version = "0.32.0"
10+
version = "0.33.0"
1111
edition = "2024"
1212
# MSRV: 1.88 — the minimum the dependency tree supports (ratatui 0.30's
1313
# `ratatui-core`/`-widgets`/`-crossterm`). Bumped from 1.85 when updating ratatui
@@ -39,19 +39,19 @@ todo = "warn"
3939
[workspace.dependencies]
4040
# Internal crates (path deps with a version so `cargo deny` sees no wildcard),
4141
# referenced as `quiver-x = { workspace = true }`.
42-
quiver-core = { path = "crates/quiver-core", version = "0.32", package = "quiverdb-core" }
43-
quiver-cluster = { path = "crates/quiver-cluster", version = "0.32", package = "quiverdb-cluster" }
44-
quiver-crypto = { path = "crates/quiver-crypto", version = "0.32", package = "quiverdb-crypto" }
45-
quiver-embed = { path = "crates/quiver-embed", version = "0.32", package = "quiverdb-embed" }
46-
quiver-import = { path = "crates/quiver-import", version = "0.32", package = "quiverdb-import" }
47-
quiver-index = { path = "crates/quiver-index", version = "0.32", package = "quiverdb-index" }
48-
quiver-mcp = { path = "crates/quiver-mcp", version = "0.32", package = "quiverdb-mcp" }
49-
quiver-proto = { path = "crates/quiver-proto", version = "0.32", package = "quiverdb-proto" }
50-
quiver-providers = { path = "crates/quiver-providers", version = "0.32", package = "quiverdb-providers" }
51-
quiver-query = { path = "crates/quiver-query", version = "0.32", package = "quiverdb-query" }
52-
quiver-server = { path = "crates/quiver-server", version = "0.32", package = "quiverdb-server" }
53-
quiver-simd = { path = "crates/quiver-simd", version = "0.32", package = "quiverdb-simd" }
54-
quiver-tui = { path = "crates/quiver-tui", version = "0.32", package = "quiverdb-tui" }
42+
quiver-core = { path = "crates/quiver-core", version = "0.33", package = "quiverdb-core" }
43+
quiver-cluster = { path = "crates/quiver-cluster", version = "0.33", package = "quiverdb-cluster" }
44+
quiver-crypto = { path = "crates/quiver-crypto", version = "0.33", package = "quiverdb-crypto" }
45+
quiver-embed = { path = "crates/quiver-embed", version = "0.33", package = "quiverdb-embed" }
46+
quiver-import = { path = "crates/quiver-import", version = "0.33", package = "quiverdb-import" }
47+
quiver-index = { path = "crates/quiver-index", version = "0.33", package = "quiverdb-index" }
48+
quiver-mcp = { path = "crates/quiver-mcp", version = "0.33", package = "quiverdb-mcp" }
49+
quiver-proto = { path = "crates/quiver-proto", version = "0.33", package = "quiverdb-proto" }
50+
quiver-providers = { path = "crates/quiver-providers", version = "0.33", package = "quiverdb-providers" }
51+
quiver-query = { path = "crates/quiver-query", version = "0.33", package = "quiverdb-query" }
52+
quiver-server = { path = "crates/quiver-server", version = "0.33", package = "quiverdb-server" }
53+
quiver-simd = { path = "crates/quiver-simd", version = "0.33", package = "quiverdb-simd" }
54+
quiver-tui = { path = "crates/quiver-tui", version = "0.33", package = "quiverdb-tui" }
5555

5656
anyhow = "1"
5757
# Lock-free atomic `Arc` swap for the MVCC serving snapshot (ADR-0064): readers

‎README.md‎

Lines changed: 3 additions & 3 deletions
Large diffs are not rendered by default.

‎bench/uv.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/quiver-core/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,5 @@ pub use error::{CoreError, Result};
4141
pub use ids::{CollectionId, Lsn};
4242
pub use keyring::{KeyRing, SingleCodecKeyRing};
4343
pub use sec::{SecPredicate, SecValue};
44-
pub use store::{CommitObserver, Record, Store};
44+
pub use store::{CommitObserver, Record, Store, VectorSource};
4545
pub use wal::{WalEntry, WalOp};

‎crates/quiver-core/src/segment.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,19 @@ pub(crate) fn open_segment(
404404
})
405405
}
406406

407+
/// Open just a sealed segment's immutable `.vec` column as a standalone `mmap`,
408+
/// for a lock-free off-lock vector stream (ADR-0070). Cheaper than
409+
/// [`open_segment`], which also loads the payload directory, tombstones, and
410+
/// secondary index a pure vector scan never touches. Rows are read at
411+
/// `row × stride` exactly as [`SealedSegment::read_vector`] does.
412+
pub(crate) fn open_vec_column(
413+
seg_dir: &Path,
414+
segment_id: u64,
415+
codec: &dyn PageCodec,
416+
) -> Result<BlockFile> {
417+
BlockFile::open(&vec_path(seg_dir, segment_id), codec, PageType::Segment)
418+
}
419+
407420
/// File name of a segment's vector column.
408421
fn vec_path(seg_dir: &Path, seg_id: u64) -> PathBuf {
409422
seg_dir.join(format!("seg-{seg_id:010}.vec"))

0 commit comments

Comments
 (0)