add columnar hnsw build path and bit-exact f32 export tests

Author: tazarov

Cargo.lock

@@ -10,6 +10,7 @@ crate-type = ["rlib", "cdylib"]
 arrow-array = "54.2"
 arrow-ipc = "54.2"
 arrow-schema = "54.2"
+fast-hnsw = "1.0.0"
 parquet = { version = "54.2", features = ["arrow"] }
 serde = { version = "1.0", features = ["derive"] }
 serde-pickle = "1.2"

Cargo.toml

@@ -25,6 +25,20 @@ Output format:
 - `parquet`
 - `arrow_ipc`
 
+## What It Builds
+
+From a columnar export (`parquet` or `arrow_ipc`) containing at least:
+
+- `vector` (fixed-size list/list of float32/float64)
+- optional `label` (otherwise falls back to `internal_id`, then row index)
+- optional `deleted` (bool/int)
+
+Build output:
+
+- A new persisted HNSW file (`.hnsw`) produced by the Rust `fast-hnsw` format.
+
+Note: this build output is not Chroma/hnswlib's native persistence layout (`header.bin`, `data_level0.bin`, ...).
+
 ## Build
 
 ```bash
@@ -49,12 +63,16 @@ Dynamic library output (platform-specific):
 Use `extract_index(...)` for record-by-record streaming callbacks, or
 `extract_index_to_columnar(...)` for file output.
 
+Use `build_index_from_columnar(...)` to build a new HNSW index file from
+Parquet/Arrow exports.
+
 ## FFI API
 
 ### Exported Symbols
 
 - `hnsw_toolbox_version() -> *const c_char`
 - `hnsw_toolbox_extract_index(request_json: *const c_char) -> *mut c_char`
+- `hnsw_toolbox_build_index(request_json: *const c_char) -> *mut c_char`
 - `hnsw_toolbox_get_last_error() -> *const c_char`
 - `hnsw_toolbox_free_string(ptr: *mut c_char)`
 
@@ -88,6 +106,44 @@ Use `extract_index(...)` for record-by-record streaming callbacks, or
 }
 ```
 
+### Build Request JSON
+
+```json
+{
+  "input_path": "/tmp/extracted.parquet",
+  "output_path": "/tmp/rebuilt.hnsw",
+  "input_format": "parquet",
+  "metric": "euclidean",
+  "include_deleted": false,
+  "m": 16,
+  "m0": 32,
+  "ef_construction": 200,
+  "batch_size": 1024,
+  "capacity": 100000,
+  "seed": 42
+}
+```
+
+`input_format`: `"parquet"` or `"arrow_ipc"` (defaults to `"parquet"`).  
+`metric`: `"euclidean"`, `"squared_euclidean"`, `"cosine"`, `"dot_product"`, `"manhattan"` (defaults to `"euclidean"`).
+
+### Build Response JSON
+
+```json
+{
+  "input_path": "/tmp/extracted.parquet",
+  "output_path": "/tmp/rebuilt.hnsw",
+  "input_format": "parquet",
+  "metric": "euclidean",
+  "summary": {
+    "scanned": 10000,
+    "inserted": 8000,
+    "deleted_skipped": 2000,
+    "dimension": 384
+  }
+}
+```
+
 ## Go Usage (Purego, No CGO)
 
 ```go
@@ -113,6 +169,25 @@ if err != nil {
 _ = resp
 ```
 
+Build a new index:
+
+```go
+buildResp, err := hnswtoolbox.BuildIndex(hnswtoolbox.BuildRequest{
+    InputPath:      "/tmp/extracted.parquet",
+    OutputPath:     "/tmp/rebuilt.hnsw",
+    InputFormat:    hnswtoolbox.InputFormatParquet,
+    Metric:         hnswtoolbox.DistanceMetricEuclidean,
+    IncludeDeleted: false,
+    M:              16,
+    EfConstruction: 200,
+    BatchSize:      1024,
+})
+if err != nil {
+    panic(err)
+}
+_ = buildResp
+```
+
 ## Validation
 
 ```bash

README.md

@@ -18,6 +18,23 @@ const (
 	OutputFormatArrowIPC OutputFormat = "arrow_ipc"
 )
 
+type InputFormat string
+
+const (
+	InputFormatParquet  InputFormat = "parquet"
+	InputFormatArrowIPC InputFormat = "arrow_ipc"
+)
+
+type DistanceMetric string
+
+const (
+	DistanceMetricEuclidean        DistanceMetric = "euclidean"
+	DistanceMetricSquaredEuclidean DistanceMetric = "squared_euclidean"
+	DistanceMetricCosine           DistanceMetric = "cosine"
+	DistanceMetricDotProduct       DistanceMetric = "dot_product"
+	DistanceMetricManhattan        DistanceMetric = "manhattan"
+)
+
 type ExtractRequest struct {
 	IndexDir       string
 	OutputPath     string
@@ -40,6 +57,35 @@ type ExtractResponse struct {
 	Summary      ExtractSummary `json:"summary"`
 }
 
+type BuildRequest struct {
+	InputPath      string
+	OutputPath     string
+	InputFormat    InputFormat
+	Metric         DistanceMetric
+	IncludeDeleted bool
+	M              int
+	M0             *int
+	EfConstruction int
+	BatchSize      int
+	Capacity       *int
+	Seed           *uint64
+}
+
+type BuildSummary struct {
+	Scanned        uint64 `json:"scanned"`
+	Inserted       uint64 `json:"inserted"`
+	DeletedSkipped uint64 `json:"deleted_skipped"`
+	Dimension      int    `json:"dimension"`
+}
+
+type BuildResponse struct {
+	InputPath   string         `json:"input_path"`
+	OutputPath  string         `json:"output_path"`
+	InputFormat InputFormat    `json:"input_format"`
+	Metric      DistanceMetric `json:"metric"`
+	Summary     BuildSummary   `json:"summary"`
+}
+
 type extractPayload struct {
 	IndexDir       string       `json:"index_dir"`
 	OutputPath     string       `json:"output_path"`
@@ -49,6 +95,20 @@ type extractPayload struct {
 	BatchSize      int          `json:"batch_size,omitempty"`
 }
 
+type buildPayload struct {
+	InputPath      string         `json:"input_path"`
+	OutputPath     string         `json:"output_path"`
+	InputFormat    InputFormat    `json:"input_format,omitempty"`
+	Metric         DistanceMetric `json:"metric,omitempty"`
+	IncludeDeleted bool           `json:"include_deleted,omitempty"`
+	M              int            `json:"m,omitempty"`
+	M0             *int           `json:"m0,omitempty"`
+	EfConstruction int            `json:"ef_construction,omitempty"`
+	BatchSize      int            `json:"batch_size,omitempty"`
+	Capacity       *int           `json:"capacity,omitempty"`
+	Seed           *uint64        `json:"seed,omitempty"`
+}
+
 var (
 	stateMu sync.Mutex
 	callMu  sync.Mutex
@@ -57,6 +117,7 @@ var (
 	loaded    bool
 
 	fnExtractIndex func(*byte) *byte
+	fnBuildIndex   func(*byte) *byte
 	fnLastError    func() *byte
 	fnFreeCString  func(*byte)
 	fnVersion      func() *byte
@@ -82,6 +143,10 @@ func Init(libraryPath string) error {
 		_ = purego.Dlclose(handle)
 		return err
 	}
+	if err := register(handle, &fnBuildIndex, "hnsw_toolbox_build_index"); err != nil {
+		_ = purego.Dlclose(handle)
+		return err
+	}
 	if err := register(handle, &fnLastError, "hnsw_toolbox_get_last_error"); err != nil {
 		_ = purego.Dlclose(handle)
 		return err
@@ -189,6 +254,59 @@ func ExtractIndex(request ExtractRequest) (*ExtractResponse, error) {
 	return &response, nil
 }
 
+func BuildIndex(request BuildRequest) (*BuildResponse, error) {
+	if err := ensureLoaded(); err != nil {
+		return nil, err
+	}
+	if strings.TrimSpace(request.InputPath) == "" {
+		return nil, errors.New("InputPath is required")
+	}
+	if strings.TrimSpace(request.OutputPath) == "" {
+		return nil, errors.New("OutputPath is required")
+	}
+
+	payload := buildPayload{
+		InputPath:      request.InputPath,
+		OutputPath:     request.OutputPath,
+		InputFormat:    request.InputFormat,
+		Metric:         request.Metric,
+		IncludeDeleted: request.IncludeDeleted,
+		M:              request.M,
+		M0:             request.M0,
+		EfConstruction: request.EfConstruction,
+		BatchSize:      request.BatchSize,
+		Capacity:       request.Capacity,
+		Seed:           request.Seed,
+	}
+
+	rawPayload, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal build request: %w", err)
+	}
+	cPayload := append(rawPayload, 0)
+
+	callMu.Lock()
+	responsePtr := fnBuildIndex(&cPayload[0])
+	if responsePtr == nil {
+		errorMessage := goStringFromPtr(fnLastError())
+		callMu.Unlock()
+		if strings.TrimSpace(errorMessage) == "" {
+			errorMessage = "hnsw_toolbox_build_index failed without error message"
+		}
+		return nil, errors.New(errorMessage)
+	}
+
+	responseJSON := goStringFromPtr(responsePtr)
+	fnFreeCString(responsePtr)
+	callMu.Unlock()
+
+	var response BuildResponse
+	if err := json.Unmarshal([]byte(responseJSON), &response); err != nil {
+		return nil, fmt.Errorf("failed to parse build response: %w", err)
+	}
+	return &response, nil
+}
+
 func isLoaded() bool {
 	stateMu.Lock()
 	defer stateMu.Unlock()