refactor: Unify index directives (#5188)

## Relevant issue(s)

Resolves #5168 

## Description

This unifies the `@vectorIndex` and `@index` directive, while
maintaining full backwards compatability, and clean cut outs for adding
new index types in the future.

## Tasks

- [x] I made sure the code is well commented, particularly
hard-to-understand areas.
- [ ] I made sure the repository-held documentation is changed
accordingly.
- [x] I made sure the pull request title adheres to the conventional
commit style (the subset used in the project can be found in
[tools/configs/chglog/config.yml](tools/configs/chglog/config.yml)).
- [x] I made sure to discuss its limitations such as threats to
validity, vulnerability to mistake and misuse, robustness to
invalidation of assumptions, resource requirements, ...

## How has this been tested?

Expanded unit and integration tests. Additional manual testing

Specify the platform(s) on which this was tested:
- Linux (nixos)

Author: jsimnz

client/index.go

@@ -82,9 +82,9 @@ type HNSWParams struct {
 	EfSearch uint32
 }
 
-// Default HNSW parameters, applied when the corresponding @vectorIndex directive argument is
-// omitted. These are the single source of truth: both the GraphQL directive definition and the
-// directive parser reference them, so the documented defaults cannot drift apart.
+// Default HNSW parameters, applied when the corresponding @index vector configuration is omitted.
+// These are the single source of truth: both the GraphQL directive definition and parser reference
+// them, so the documented defaults cannot drift apart.
 const (
 	// DefaultHNSWM is the default maximum number of connections per node. Higher values improve
 	// recall at the cost of memory and build time.
@@ -121,8 +121,7 @@ type VectorIndexDescription struct {
 	Algorithm VectorAlgorithm
 	// Metric is the distance metric used to compare vectors.
 	Metric DistanceMetric
-	// Dimensions is the length of the vectors being indexed. It must be set, except on an @embedding
-	// field, where the embedding model fixes the length and Dimensions may be left 0.
+	// Dimensions is the length of the vectors being indexed. It must be greater than zero.
 	Dimensions uint32
 	// HNSW holds HNSW-specific parameters. Non-nil when Algorithm == VectorAlgorithmHNSW.
 	HNSW *HNSWParams

internal/db/collection_index.go

@@ -559,10 +559,9 @@ func processNewIndexRequest(
 }
 
 // validateVectorIndexDescription checks and fills in the vector-specific parts of an index request.
-// The field must hold a float32 array, and dimensions must be set unless the field is an @embedding,
-// whose model fixes the vector length. It also defaults the algorithm, metric, and any missing
-// params (mutating desc.Vector), so a request made through the index API works the same as one from
-// the @vectorIndex directive.
+// The field must hold a float32 array, and dimensions must be greater than zero. It also defaults
+// the algorithm, metric, and any missing params (mutating desc.Vector), so a request made through
+// the index API works the same as one from the @index directive's vector configuration.
 //
 // The field is guaranteed to exist here because validateIndexDescription and
 // checkExistingFieldsAndAdjustRelFieldNames run before this and already check that.
@@ -583,10 +582,13 @@ func validateVectorIndexDescription(def client.CollectionVersion, desc client.Ne
 	if !client.IsVectorEmbeddingCompatible(field.Kind) {
 		return NewErrUnsupportedVectorIndexFieldType(field.Kind)
 	}
+	if desc.Vector.Dimensions == 0 {
+		return NewErrVectorIndexMissingDimensions(fieldName)
+	}
 
-	// The config object present is what picks the algorithm, so the caller never sets one directly.
-	// Fill in the algorithm, metric, and any missing params with defaults. A nil config means an empty
-	// one, so a caller can leave it out and still get a working HNSW index.
+	// The algorithm config object present is what picks the algorithm, so the caller never sets one
+	// directly. Fill in the algorithm, metric, and any missing params with defaults. A nil HNSW config
+	// means an empty one, so a caller can leave it out and still get a working HNSW index.
 	if desc.Vector.HNSW == nil {
 		desc.Vector.HNSW = &client.HNSWParams{}
 	}
@@ -612,19 +614,7 @@ func validateVectorIndexDescription(def client.CollectionVersion, desc client.Ne
 		return err
 	}
 
-	if desc.Vector.Dimensions > 0 {
-		return nil
-	}
-
-	// No dimensions were given. That is only allowed when the field is an @embedding, since the
-	// model then fixes the dimensions. The value itself is filled in later.
-	for _, embedding := range def.VectorEmbeddings {
-		if embedding.FieldName == fieldName {
-			return nil
-		}
-	}
-
-	return NewErrVectorIndexMissingDimensions(fieldName)
+	return nil
 }
 
 // validateNoConflictingVectorIndexMetric rejects creating a vector index on a field that another

internal/db/errors.go

@@ -62,7 +62,7 @@ const (
 	errInvalidFieldValue                         string = "invalid field value"
 	errUnsupportedIndexFieldType                 string = "unsupported index field type"
 	errUnsupportedVectorIndexFieldType           string = "unsupported field type for vector index"
-	errVectorIndexMissingDimensions              string = "vector index requires dimensions unless field is an embedding"
+	errVectorIndexMissingDimensions              string = "vector index dimensions must be greater than zero"
 	errCannotIndexAccumulatedCRDTField           string = "indexing accumulated CRDT fields is not yet supported"
 	errIndexDescriptionHasNoFields               string = "index description has no fields"
 	errCreateFile                                string = "failed to create file"
@@ -615,8 +615,8 @@ func NewErrUnsupportedVectorIndexFieldType(kind client.FieldKind) error {
 	)
 }
 
-// NewErrVectorIndexMissingDimensions returns a new error indicating that a vector index request is
-// missing its dimensions, and dimensions could not be inferred from a generated embedding.
+// NewErrVectorIndexMissingDimensions returns a new error indicating that a vector index request has
+// no dimensions.
 func NewErrVectorIndexMissingDimensions(fieldName string) error {
 	return errors.New(
 		errVectorIndexMissingDimensions,

internal/db/vector_index_test.go

@@ -24,7 +24,7 @@ import (
 )
 
 // newVectorIndexTestDB opens an in-memory badger-backed DB with a collection carrying a
-// [Float32!] @vectorIndex field, ready for document writes.
+// [Float32!] @index(vector: {...}) field, ready for document writes.
 func newVectorIndexTestDB(t *testing.T, dimensions int) (context.Context, *DB, client.Collection) {
 	t.Helper()
 	ctx := context.Background()
@@ -35,7 +35,7 @@ func newVectorIndexTestDB(t *testing.T, dimensions int) (context.Context, *DB, c
 	_, err = db.AddCollection(ctx, `
 		type Users {
 			name: String
-			embedding: [Float32!] @vectorIndex(dimensions: `+strconv.Itoa(dimensions)+`, HNSW: {metric: COSINE})
+			embedding: [Float32!] @index(vector: {dimensions: `+strconv.Itoa(dimensions)+`, hnsw: {metric: COSINE}})
 		}
 	`)
 	require.NoError(t, err)
@@ -82,6 +82,21 @@ func vectorIndexSearch(
 	return docIDs
 }
 
+func TestValidateVectorIndexDescription_EmbeddingRequiresDimensions(t *testing.T) {
+	const fieldName = "embedding"
+	def := client.CollectionVersion{
+		Fields:           []client.CollectionFieldDescription{{Name: fieldName, Kind: client.FieldKind_FLOAT32_ARRAY}},
+		VectorEmbeddings: []client.VectorEmbeddingDescription{{FieldName: fieldName}},
+	}
+	desc := client.NewIndexRequest{
+		Fields: []client.IndexedFieldDescription{{Name: fieldName}},
+		Vector: &client.VectorIndexDescription{},
+	}
+
+	err := validateVectorIndexDescription(def, desc)
+	require.ErrorContains(t, err, "vector index dimensions must be greater than zero")
+}
+
 func TestCollectionVectorIndex_Save_InsertsIntoGraphAndIsSearchable(t *testing.T) {
 	ctx, db, col := newVectorIndexTestDB(t, 3)