diff --git a/client/index.go b/client/index.go index fe04c6d1ca..db8c10bc7f 100644 --- a/client/index.go +++ b/client/index.go @@ -82,9 +82,9 @@ type HNSWParams struct { EfSearch uint32 } -// Default HNSW parameters, applied when the corresponding @vectorIndex directive argument is -// omitted. These are the single source of truth: both the GraphQL directive definition and the -// directive parser reference them, so the documented defaults cannot drift apart. +// Default HNSW parameters, applied when the corresponding @index vector configuration is omitted. +// These are the single source of truth: both the GraphQL directive definition and parser reference +// them, so the documented defaults cannot drift apart. const ( // DefaultHNSWM is the default maximum number of connections per node. Higher values improve // recall at the cost of memory and build time. @@ -121,8 +121,7 @@ type VectorIndexDescription struct { Algorithm VectorAlgorithm // Metric is the distance metric used to compare vectors. Metric DistanceMetric - // Dimensions is the length of the vectors being indexed. It must be set, except on an @embedding - // field, where the embedding model fixes the length and Dimensions may be left 0. + // Dimensions is the length of the vectors being indexed. It must be greater than zero. Dimensions uint32 // HNSW holds HNSW-specific parameters. Non-nil when Algorithm == VectorAlgorithmHNSW. HNSW *HNSWParams diff --git a/internal/db/collection_index.go b/internal/db/collection_index.go index 4c657d629a..0f8285213b 100644 --- a/internal/db/collection_index.go +++ b/internal/db/collection_index.go @@ -559,10 +559,9 @@ func processNewIndexRequest( } // validateVectorIndexDescription checks and fills in the vector-specific parts of an index request. -// The field must hold a float32 array, and dimensions must be set unless the field is an @embedding, -// whose model fixes the vector length. It also defaults the algorithm, metric, and any missing -// params (mutating desc.Vector), so a request made through the index API works the same as one from -// the @vectorIndex directive. +// The field must hold a float32 array, and dimensions must be greater than zero. It also defaults +// the algorithm, metric, and any missing params (mutating desc.Vector), so a request made through +// the index API works the same as one from the @index directive's vector configuration. // // The field is guaranteed to exist here because validateIndexDescription and // checkExistingFieldsAndAdjustRelFieldNames run before this and already check that. @@ -583,10 +582,13 @@ func validateVectorIndexDescription(def client.CollectionVersion, desc client.Ne if !client.IsVectorEmbeddingCompatible(field.Kind) { return NewErrUnsupportedVectorIndexFieldType(field.Kind) } + if desc.Vector.Dimensions == 0 { + return NewErrVectorIndexMissingDimensions(fieldName) + } - // The config object present is what picks the algorithm, so the caller never sets one directly. - // Fill in the algorithm, metric, and any missing params with defaults. A nil config means an empty - // one, so a caller can leave it out and still get a working HNSW index. + // The algorithm config object present is what picks the algorithm, so the caller never sets one + // directly. Fill in the algorithm, metric, and any missing params with defaults. A nil HNSW config + // means an empty one, so a caller can leave it out and still get a working HNSW index. if desc.Vector.HNSW == nil { desc.Vector.HNSW = &client.HNSWParams{} } @@ -612,19 +614,7 @@ func validateVectorIndexDescription(def client.CollectionVersion, desc client.Ne return err } - if desc.Vector.Dimensions > 0 { - return nil - } - - // No dimensions were given. That is only allowed when the field is an @embedding, since the - // model then fixes the dimensions. The value itself is filled in later. - for _, embedding := range def.VectorEmbeddings { - if embedding.FieldName == fieldName { - return nil - } - } - - return NewErrVectorIndexMissingDimensions(fieldName) + return nil } // validateNoConflictingVectorIndexMetric rejects creating a vector index on a field that another diff --git a/internal/db/errors.go b/internal/db/errors.go index 27dc82c50b..01f194e8f6 100644 --- a/internal/db/errors.go +++ b/internal/db/errors.go @@ -62,7 +62,7 @@ const ( errInvalidFieldValue string = "invalid field value" errUnsupportedIndexFieldType string = "unsupported index field type" errUnsupportedVectorIndexFieldType string = "unsupported field type for vector index" - errVectorIndexMissingDimensions string = "vector index requires dimensions unless field is an embedding" + errVectorIndexMissingDimensions string = "vector index dimensions must be greater than zero" errCannotIndexAccumulatedCRDTField string = "indexing accumulated CRDT fields is not yet supported" errIndexDescriptionHasNoFields string = "index description has no fields" errCreateFile string = "failed to create file" @@ -615,8 +615,8 @@ func NewErrUnsupportedVectorIndexFieldType(kind client.FieldKind) error { ) } -// NewErrVectorIndexMissingDimensions returns a new error indicating that a vector index request is -// missing its dimensions, and dimensions could not be inferred from a generated embedding. +// NewErrVectorIndexMissingDimensions returns a new error indicating that a vector index request has +// no dimensions. func NewErrVectorIndexMissingDimensions(fieldName string) error { return errors.New( errVectorIndexMissingDimensions, diff --git a/internal/db/vector_index_test.go b/internal/db/vector_index_test.go index d9994fc8a5..f401dc5388 100644 --- a/internal/db/vector_index_test.go +++ b/internal/db/vector_index_test.go @@ -24,7 +24,7 @@ import ( ) // newVectorIndexTestDB opens an in-memory badger-backed DB with a collection carrying a -// [Float32!] @vectorIndex field, ready for document writes. +// [Float32!] @index(vector: {...}) field, ready for document writes. func newVectorIndexTestDB(t *testing.T, dimensions int) (context.Context, *DB, client.Collection) { t.Helper() ctx := context.Background() @@ -35,7 +35,7 @@ func newVectorIndexTestDB(t *testing.T, dimensions int) (context.Context, *DB, c _, err = db.AddCollection(ctx, ` type Users { name: String - embedding: [Float32!] @vectorIndex(dimensions: `+strconv.Itoa(dimensions)+`, HNSW: {metric: COSINE}) + embedding: [Float32!] @index(vector: {dimensions: `+strconv.Itoa(dimensions)+`, hnsw: {metric: COSINE}}) } `) require.NoError(t, err) @@ -82,6 +82,21 @@ func vectorIndexSearch( return docIDs } +func TestValidateVectorIndexDescription_EmbeddingRequiresDimensions(t *testing.T) { + const fieldName = "embedding" + def := client.CollectionVersion{ + Fields: []client.CollectionFieldDescription{{Name: fieldName, Kind: client.FieldKind_FLOAT32_ARRAY}}, + VectorEmbeddings: []client.VectorEmbeddingDescription{{FieldName: fieldName}}, + } + desc := client.NewIndexRequest{ + Fields: []client.IndexedFieldDescription{{Name: fieldName}}, + Vector: &client.VectorIndexDescription{}, + } + + err := validateVectorIndexDescription(def, desc) + require.ErrorContains(t, err, "vector index dimensions must be greater than zero") +} + func TestCollectionVectorIndex_Save_InsertsIntoGraphAndIsSearchable(t *testing.T) { ctx, db, col := newVectorIndexTestDB(t, 3) diff --git a/internal/request/graphql/schema/collection.go b/internal/request/graphql/schema/collection.go index 0ebac4cc13..8eeb493f12 100644 --- a/internal/request/graphql/schema/collection.go +++ b/internal/request/graphql/schema/collection.go @@ -155,12 +155,6 @@ func fromAstDefinition( return core.Collection{}, err } encryptedIndexes = append(encryptedIndexes, encryptedIndex) - case types.VectorIndexDirectiveLabel: - index, err := vectorIndexFromAST(directive, field) - if err != nil { - return core.Collection{}, err - } - indexes = append(indexes, index) } } } @@ -265,57 +259,162 @@ func IsValidIndexName(name string) bool { return true } -func indexFromAST(directive *ast.Directive, fieldDef *ast.FieldDefinition) (client.NewIndexRequest, error) { - var name string - var unique bool +type orderedIndexConfig struct { + unique bool + direction *ast.EnumValue + includes *ast.ListValue + hasUnique bool + hasDirection bool + hasIncludes bool +} - var direction *ast.EnumValue - var includes *ast.ListValue +type indexDirectiveConfig struct { + name string + kind string + + // orderedIndexConfig is broken out to support the legacy and newer config + // options. New and future index types should use the same approach as the + // `vector` field and just have the single `ast.Value` that is directly + // parsed. + ordered orderedIndexConfig + vector ast.Value +} + +func (c *indexDirectiveConfig) selectKind(kind string) error { + if c.kind != "" && c.kind != kind { + return ErrIndexWithInvalidArg + } + c.kind = kind + return nil +} + +func (c indexDirectiveConfig) newIndex(fieldDef *ast.FieldDefinition) (client.NewIndexRequest, error) { + switch c.kind { + case "", types.OrderedIndexKind: + return orderedIndexFromConfig(c.name, c.ordered, fieldDef) + case types.VectorIndexKind: + return vectorIndexFromAST(c.name, c.vector, fieldDef) + default: + return client.NewIndexRequest{}, ErrIndexWithInvalidArg + } +} + +func indexFromAST(directive *ast.Directive, fieldDef *ast.FieldDefinition) (client.NewIndexRequest, error) { + var config indexDirectiveConfig for _, arg := range directive.Arguments { switch arg.Name.Value { case types.IndexDirectivePropName: - nameVal, ok := arg.Value.(*ast.StringValue) + name, ok := arg.Value.(*ast.StringValue) if !ok { return client.NewIndexRequest{}, ErrIndexWithInvalidArg } - name = nameVal.Value - if !IsValidIndexName(name) { - return client.NewIndexRequest{}, NewErrIndexWithInvalidName(name) + if !IsValidIndexName(name.Value) { + return client.NewIndexRequest{}, NewErrIndexWithInvalidName(name.Value) } + config.name = name.Value - case types.IndexDirectivePropIncludes: - includesVal, ok := arg.Value.(*ast.ListValue) + case types.IndexDirectivePropKind: + kind, ok := arg.Value.(*ast.EnumValue) if !ok { return client.NewIndexRequest{}, ErrIndexWithInvalidArg } - includes = includesVal + if err := config.selectKind(kind.Value); err != nil { + return client.NewIndexRequest{}, err + } - case types.IndexDirectivePropDirection: - directionVal, ok := arg.Value.(*ast.EnumValue) - if !ok { - return client.NewIndexRequest{}, ErrIndexWithInvalidArg + case types.OrderedIndexKind: + if err := config.selectKind(types.OrderedIndexKind); err != nil { + return client.NewIndexRequest{}, err + } + if err := parseOrderedIndexConfig(arg.Value, &config.ordered); err != nil { + return client.NewIndexRequest{}, err } - direction = directionVal - case types.IndexDirectivePropUnique: - uniqueVal, ok := arg.Value.(*ast.BooleanValue) - if !ok { - return client.NewIndexRequest{}, ErrIndexWithInvalidArg + case types.VectorIndexKind: + if err := config.selectKind(types.VectorIndexKind); err != nil { + return client.NewIndexRequest{}, err } - unique = uniqueVal.Value + config.vector = arg.Value default: - return client.NewIndexRequest{}, ErrIndexWithUnknownArg + if err := config.selectKind(types.OrderedIndexKind); err != nil { + return client.NewIndexRequest{}, err + } + if err := parseOrderedIndexProperty(arg.Name.Value, arg.Value, &config.ordered); err != nil { + return client.NewIndexRequest{}, err + } } } + return config.newIndex(fieldDef) +} + +func parseOrderedIndexConfig(value ast.Value, config *orderedIndexConfig) error { + obj, ok := value.(*ast.ObjectValue) + if !ok { + return ErrIndexWithInvalidArg + } + for _, field := range obj.Fields { + if err := parseOrderedIndexProperty(field.Name.Value, field.Value, config); err != nil { + return err + } + } + return nil +} + +func parseOrderedIndexProperty(name string, value ast.Value, config *orderedIndexConfig) error { + switch name { + case types.IndexDirectivePropIncludes: + if config.hasIncludes { + return ErrIndexWithInvalidArg + } + includes, ok := value.(*ast.ListValue) + if !ok { + return ErrIndexWithInvalidArg + } + config.includes = includes + config.hasIncludes = true + + case types.IndexDirectivePropDirection: + if config.hasDirection { + return ErrIndexWithInvalidArg + } + direction, ok := value.(*ast.EnumValue) + if !ok { + return ErrIndexWithInvalidArg + } + config.direction = direction + config.hasDirection = true + + case types.IndexDirectivePropUnique: + if config.hasUnique { + return ErrIndexWithInvalidArg + } + unique, ok := value.(*ast.BooleanValue) + if !ok { + return ErrIndexWithInvalidArg + } + config.unique = unique.Value + config.hasUnique = true + + default: + return ErrIndexWithUnknownArg + } + return nil +} + +func orderedIndexFromConfig( + name string, + config orderedIndexConfig, + fieldDef *ast.FieldDefinition, +) (client.NewIndexRequest, error) { var containsField bool var fields []client.IndexedFieldDescription - if includes != nil { - for _, include := range includes.Values { - field, err := indexFieldFromAST(include, direction) + if config.includes != nil { + for _, include := range config.includes.Values { + field, err := indexFieldFromAST(include, config.direction) if err != nil { return client.NewIndexRequest{}, err } @@ -326,15 +425,11 @@ func indexFromAST(directive *ast.Directive, fieldDef *ast.FieldDefinition) (clie } } - // if the directive is applied to a field and - // the field is not in the includes list - // implicitly add it as the first entry + // If the directive is applied to a field that is not in the includes list, add it first. if !containsField && fieldDef != nil { - field := client.IndexedFieldDescription{ - Name: fieldDef.Name.Value, - } - if direction != nil { - field.Descending = direction.Value == types.FieldOrderDESC + field := client.IndexedFieldDescription{Name: fieldDef.Name.Value} + if config.direction != nil { + field.Descending = config.direction.Value == types.FieldOrderDESC } fields = append([]client.IndexedFieldDescription{field}, fields...) } @@ -346,7 +441,7 @@ func indexFromAST(directive *ast.Directive, fieldDef *ast.FieldDefinition) (clie return client.NewIndexRequest{ Name: name, Fields: fields, - Unique: unique, + Unique: config.unique, }, nil } @@ -648,37 +743,46 @@ func policyFromAST(directive *ast.Directive) (client.PolicyDescription, error) { } func vectorIndexFromAST( - directive *ast.Directive, + name string, + config ast.Value, fieldDef *ast.FieldDefinition, ) (client.NewIndexRequest, error) { + if fieldDef == nil { + return client.NewIndexRequest{}, ErrIndexWithInvalidArg + } + + obj, ok := config.(*ast.ObjectValue) + if !ok { + return client.NewIndexRequest{}, ErrIndexWithInvalidArg + } + var dimensions uint32 - // The algorithm is chosen by which config arg is set. HNSW is the only one today, and also the - // default when no config is given. algorithm := client.VectorAlgorithmHNSW metric := client.DistanceMetricCosine - // Defaults come from client so the directive definition and the parser cannot drift. hnswParams := client.HNSWParams{ M: client.DefaultHNSWM, EfConstruction: client.DefaultHNSWEfConstruction, EfSearch: client.DefaultHNSWEfSearch, } - - for _, arg := range directive.Arguments { - switch arg.Name.Value { - case types.VectorIndexDirectivePropDimensions: - dimensionsVal, ok := arg.Value.(*ast.IntValue) - if !ok { - return client.NewIndexRequest{}, ErrIndexWithInvalidArg - } - parsed, err := strconv.ParseUint(dimensionsVal.Value, 10, 32) + for _, field := range obj.Fields { + switch field.Name.Value { + case types.VectorIndexPropDimensions: + parsed, err := parseUint32ASTValue(field.Value) if err != nil { + return client.NewIndexRequest{}, err + } + dimensions = parsed + + case types.VectorIndexPropAlgorithm: + algorithmVal, ok := field.Value.(*ast.EnumValue) + if !ok || algorithmVal.Value != types.VectorIndexAlgorithmHNSW { return client.NewIndexRequest{}, ErrIndexWithInvalidArg } - dimensions = uint32(parsed) + algorithm = client.VectorAlgorithmHNSW - case types.VectorIndexDirectivePropHNSW: + case types.VectorIndexPropHNSW: algorithm = client.VectorAlgorithmHNSW - if err := parseHNSWConfig(arg.Value, &metric, &hnswParams); err != nil { + if err := parseHNSWConfig(field.Value, &metric, &hnswParams); err != nil { return client.NewIndexRequest{}, err } @@ -697,16 +801,13 @@ func vectorIndexFromAST( } return client.NewIndexRequest{ - Name: "", - Fields: []client.IndexedFieldDescription{ - {Name: fieldDef.Name.Value}, - }, + Name: name, + Fields: []client.IndexedFieldDescription{{Name: fieldDef.Name.Value}}, Vector: &vectorDesc, }, nil } -// parseHNSWConfig reads the @vectorIndex `HNSW` config object into metric + params, overwriting only -// the fields the user set (the rest keep their defaults). +// parseHNSWConfig reads the @index vector HNSW config, overwriting only explicitly set defaults. func parseHNSWConfig(value ast.Value, metric *client.DistanceMetric, params *client.HNSWParams) error { obj, ok := value.(*ast.ObjectValue) if !ok { @@ -731,21 +832,21 @@ func parseHNSWConfig(value ast.Value, metric *client.DistanceMetric, params *cli return NewErrVectorIndexUnknownMetric(metricVal.Value) } - case types.VectorIndexConfigPropM: + case types.VectorIndexHNSWConfigPropM: parsed, err := parseUint32ASTValue(field.Value) if err != nil { return err } params.M = parsed - case types.VectorIndexConfigPropEfConstruction: + case types.VectorIndexHNSWConfigPropEfConstruction: parsed, err := parseUint32ASTValue(field.Value) if err != nil { return err } params.EfConstruction = parsed - case types.VectorIndexConfigPropEfSearch: + case types.VectorIndexHNSWConfigPropEfSearch: parsed, err := parseUint32ASTValue(field.Value) if err != nil { return err diff --git a/internal/request/graphql/schema/index_parse_test.go b/internal/request/graphql/schema/index_parse_test.go index ba99aa53e3..bd241661c3 100644 --- a/internal/request/graphql/schema/index_parse_test.go +++ b/internal/request/graphql/schema/index_parse_test.go @@ -114,6 +114,27 @@ func TestParseIndexOnStruct(t *testing.T) { }, }, }, + { + description: "Nested ordered config is equivalent to legacy ordered config", + sdl: `type user @index(name: "userIndex", ordered: { + unique: true, + direction: DESC, + includes: [{field: "name"}, {field: "age", direction: ASC}] + }) { + name: String + age: Int + }`, + targetDescriptions: []client.NewIndexRequest{ + { + Name: "userIndex", + Fields: []client.IndexedFieldDescription{ + {Name: "name", Descending: true}, + {Name: "age"}, + }, + Unique: true, + }, + }, + }, } for _, test := range cases { @@ -319,6 +340,54 @@ func TestParseIndexOnField(t *testing.T) { }, }, }, + { + description: "ordered kind uses field index defaults", + sdl: `type user { + name: String @index(kind: ordered) + }`, + targetDescriptions: []client.NewIndexRequest{ + { + Fields: []client.IndexedFieldDescription{ + {Name: "name"}, + }, + }, + }, + }, + { + description: "matching kind and ordered config are allowed", + sdl: `type user { + name: String @index(kind: ordered, ordered: {direction: DESC}) + }`, + targetDescriptions: []client.NewIndexRequest{ + { + Fields: []client.IndexedFieldDescription{{Name: "name", Descending: true}}, + }, + }, + }, + { + description: "matching kind and legacy ordered config are allowed", + sdl: `type user { + name: String @index(kind: ordered, unique: true) + }`, + targetDescriptions: []client.NewIndexRequest{ + { + Fields: []client.IndexedFieldDescription{{Name: "name"}}, + Unique: true, + }, + }, + }, + { + description: "nested and legacy ordered configs merge when they do not overlap", + sdl: `type user { + name: String @index(ordered: {unique: true}, direction: DESC) + }`, + targetDescriptions: []client.NewIndexRequest{ + { + Fields: []client.IndexedFieldDescription{{Name: "name", Descending: true}}, + Unique: true, + }, + }, + }, } for _, test := range cases { @@ -377,6 +446,27 @@ func TestParseInvalidIndexOnField(t *testing.T) { }`, expectedErr: `Argument "unique" has invalid value "true"`, }, + { + description: "ordered kind conflicts with vector config", + sdl: `type user { + name: String @index(kind: ordered, vector: {}) + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "ordered and vector configs are competing kind selectors", + sdl: `type user { + name: String @index(ordered: {}, vector: {}) + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "nested and legacy ordered configs cannot set the same property", + sdl: `type user { + name: String @index(ordered: {unique: true}, unique: false) + }`, + expectedErr: errIndexInvalidArgument, + }, } for _, test := range cases { diff --git a/internal/request/graphql/schema/schema.go b/internal/request/graphql/schema/schema.go index d718fbf932..0bfe8c743b 100644 --- a/internal/request/graphql/schema/schema.go +++ b/internal/request/graphql/schema/schema.go @@ -19,11 +19,14 @@ import ( // defaultSchema returns a new gql.Schema containing the default type definitions. func defaultSchema() (gql.Schema, error) { orderEnum := types.OrderingEnum() + indexKindEnum := types.IndexKindEnum() commitsEnum := types.CommitsEnum() crdtEnum := types.CRDTEnum() explainEnum := types.ExplainEnum() vectorDistanceMetricEnum := types.VectorDistanceMetricEnum() + vectorIndexAlgorithmEnum := types.VectorIndexAlgorithmEnum() hnswIndexConfigInput := types.HNSWIndexConfigInputObject(vectorDistanceMetricEnum) + vectorIndexInput := types.VectorIndexInputObject(hnswIndexConfigInput, vectorIndexAlgorithmEnum) commitsOrderArg := types.CommitsOrderArg(orderEnum) commitsFilterFieldNameArg := types.CommitsFilterFieldNameArg() @@ -34,6 +37,7 @@ func defaultSchema() (gql.Schema, error) { encryptedSearchResult := types.EncryptedSearchResultObject() indexFieldInput := types.IndexFieldInputObject(orderEnum) + orderedIndexInput := types.OrderedIndexInputObject(orderEnum, indexFieldInput) queryCommits := types.QueryCommits(commitObject, commitsOrderArg, commitsFilterArg, commitsEnum) @@ -43,12 +47,16 @@ func defaultSchema() (gql.Schema, error) { commitsOrderArg, commitsEnum, orderEnum, + indexKindEnum, crdtEnum, explainEnum, indexFieldInput, + orderedIndexInput, encryptedSearchResult, + vectorIndexAlgorithmEnum, vectorDistanceMetricEnum, hnswIndexConfigInput, + vectorIndexInput, ), Query: defaultQueryType(queryCommits), Mutation: defaultMutationType(), @@ -57,7 +65,9 @@ func defaultSchema() (gql.Schema, error) { explainEnum, orderEnum, indexFieldInput, - hnswIndexConfigInput, + indexKindEnum, + orderedIndexInput, + vectorIndexInput, ), Subscription: defaultSubscriptionType(queryCommits), }) @@ -103,7 +113,9 @@ func defaultDirectivesType( explainEnum *gql.Enum, orderEnum *gql.Enum, indexFieldInput *gql.InputObject, - hnswIndexConfigInput *gql.InputObject, + indexKindEnum *gql.Enum, + orderedIndexInput *gql.InputObject, + vectorIndexInput *gql.InputObject, ) []*gql.Directive { return []*gql.Directive{ types.CRDTFieldDirective(crdtEnum), @@ -111,7 +123,7 @@ func defaultDirectivesType( types.ExhaustiveDirective(), types.ExplainDirective(explainEnum), types.PolicyDirective(), - types.IndexDirective(orderEnum, indexFieldInput), + types.IndexDirective(orderEnum, indexFieldInput, indexKindEnum, orderedIndexInput, vectorIndexInput), types.PrimaryDirective(), types.RelationDirective(), types.MaterializedDirective(), @@ -119,7 +131,6 @@ func defaultDirectivesType( types.VectorEmbeddingDirective(), types.ConstraintsDirective(), types.EncryptedIndexDirective(), - types.VectorIndexDirective(hnswIndexConfigInput), } } @@ -146,12 +157,16 @@ func defaultTypes( commitsOrderArg *gql.InputObject, commitsEnum *gql.Enum, orderEnum *gql.Enum, + indexKindEnum *gql.Enum, crdtEnum *gql.Enum, explainEnum *gql.Enum, indexFieldInput *gql.InputObject, + orderedIndexInput *gql.InputObject, encryptedSearchResult *gql.Object, + vectorIndexAlgorithmEnum *gql.Enum, vectorDistanceMetricEnum *gql.Enum, hnswIndexConfigInput *gql.InputObject, + vectorIndexInput *gql.InputObject, ) []gql.Type { idOpBlock := types.IDOperatorBlock() intOpBlock := types.IntOperatorBlock() @@ -191,6 +206,7 @@ func defaultTypes( // Sort/Order enum orderEnum, + indexKindEnum, // Filter scalar blocks idOpBlock, @@ -238,9 +254,12 @@ func defaultTypes( explainEnum, indexFieldInput, + orderedIndexInput, encryptedSearchResult, + vectorIndexAlgorithmEnum, vectorDistanceMetricEnum, hnswIndexConfigInput, + vectorIndexInput, } } diff --git a/internal/request/graphql/schema/testfixtures/schema.relatedmany.gen.graphql b/internal/request/graphql/schema/testfixtures/schema.relatedmany.gen.graphql index 89a8b5d4eb..a8e1399f00 100644 --- a/internal/request/graphql/schema/testfixtures/schema.relatedmany.gen.graphql +++ b/internal/request/graphql/schema/testfixtures/schema.relatedmany.gen.graphql @@ -1547,6 +1547,11 @@ input IndexField { field: String } +enum IndexKind { + "ordered scalar index" + ordered +} + "The `Int` scalar type represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1. " scalar Int @@ -2565,6 +2570,26 @@ input NotNullStringOperatorBlock { _nlike: String } +"Configures an ordered index over one or more scalar fields." +input OrderedIndexConfig { + """ + Sets the default index ordering for all fields. + + If a field in the includes list does not specify a direction + the default ordering from this value will be used instead. + """ + direction: Ordering + """ + Sets the fields the index is added on. + + When used on a field definition and the field is not in the includes list + it will be implicitly added as the first entry. + """ + includes: [IndexField] + "Makes the index unique." + unique: Boolean +} + enum Ordering { """ @@ -3179,6 +3204,21 @@ enum VectorDistanceMetric { EUCLIDEAN } +enum VectorIndexAlgorithm { + "HNSW (Hierarchical Navigable Small World)" + hnsw +} + +"Configures an approximate-nearest-neighbour index over a vector field." +input VectorIndexConfig { + "Selects the vector index algorithm using its default configuration." + alg: VectorIndexAlgorithm + "Vector dimensions; must be greater than zero." + dimensions: Int + "Configures the HNSW algorithm." + hnsw: HNSWIndexConfig +} + """ A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document. @@ -3483,7 +3523,7 @@ directive @explain( directive @index( """ Sets the default index ordering for all fields. - + If a field in the includes list does not specify a direction the default ordering from this value will be used instead. """ @@ -3495,10 +3535,16 @@ directive @index( it will be implicitly added as the first entry. """ includes: [IndexField] + "Selects an index kind using its default configuration." + kind: IndexKind "Sets the index name." name: String + "Configures an ordered index." + ordered: OrderedIndexConfig "Makes the index unique." unique: Boolean + "Configures a vector index." + vector: VectorIndexConfig ) on OBJECT | FIELD_DEFINITION """ @@ -3536,12 +3582,4 @@ directive @relation( """ name: String -) on FIELD_DEFINITION - -"@vectorIndex builds an approximate-nearest-neighbour index over a vector field." -directive @vectorIndex( - "Build the index with the HNSW algorithm using these parameters." - HNSW: HNSWIndexConfig - "Vector dimensions; required unless inferable from an @embedding." - dimensions: Int ) on FIELD_DEFINITION \ No newline at end of file diff --git a/internal/request/graphql/schema/testfixtures/schema.relatedone.gen.graphql b/internal/request/graphql/schema/testfixtures/schema.relatedone.gen.graphql index 1b5849e800..e3e816bfaf 100644 --- a/internal/request/graphql/schema/testfixtures/schema.relatedone.gen.graphql +++ b/internal/request/graphql/schema/testfixtures/schema.relatedone.gen.graphql @@ -1490,6 +1490,11 @@ input IndexField { field: String } +enum IndexKind { + "ordered scalar index" + ordered +} + "The `Int` scalar type represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1. " scalar Int @@ -2508,6 +2513,26 @@ input NotNullStringOperatorBlock { _nlike: String } +"Configures an ordered index over one or more scalar fields." +input OrderedIndexConfig { + """ + Sets the default index ordering for all fields. + + If a field in the includes list does not specify a direction + the default ordering from this value will be used instead. + """ + direction: Ordering + """ + Sets the fields the index is added on. + + When used on a field definition and the field is not in the includes list + it will be implicitly added as the first entry. + """ + includes: [IndexField] + "Makes the index unique." + unique: Boolean +} + enum Ordering { """ @@ -3122,6 +3147,21 @@ enum VectorDistanceMetric { EUCLIDEAN } +enum VectorIndexAlgorithm { + "HNSW (Hierarchical Navigable Small World)" + hnsw +} + +"Configures an approximate-nearest-neighbour index over a vector field." +input VectorIndexConfig { + "Selects the vector index algorithm using its default configuration." + alg: VectorIndexAlgorithm + "Vector dimensions; must be greater than zero." + dimensions: Int + "Configures the HNSW algorithm." + hnsw: HNSWIndexConfig +} + """ A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document. @@ -3426,7 +3466,7 @@ directive @explain( directive @index( """ Sets the default index ordering for all fields. - + If a field in the includes list does not specify a direction the default ordering from this value will be used instead. """ @@ -3438,10 +3478,16 @@ directive @index( it will be implicitly added as the first entry. """ includes: [IndexField] + "Selects an index kind using its default configuration." + kind: IndexKind "Sets the index name." name: String + "Configures an ordered index." + ordered: OrderedIndexConfig "Makes the index unique." unique: Boolean + "Configures a vector index." + vector: VectorIndexConfig ) on OBJECT | FIELD_DEFINITION """ @@ -3479,12 +3525,4 @@ directive @relation( """ name: String -) on FIELD_DEFINITION - -"@vectorIndex builds an approximate-nearest-neighbour index over a vector field." -directive @vectorIndex( - "Build the index with the HNSW algorithm using these parameters." - HNSW: HNSWIndexConfig - "Vector dimensions; required unless inferable from an @embedding." - dimensions: Int ) on FIELD_DEFINITION \ No newline at end of file diff --git a/internal/request/graphql/schema/testfixtures/schema.simple.gen.graphql b/internal/request/graphql/schema/testfixtures/schema.simple.gen.graphql index e6c1362c30..f65a8aa653 100644 --- a/internal/request/graphql/schema/testfixtures/schema.simple.gen.graphql +++ b/internal/request/graphql/schema/testfixtures/schema.simple.gen.graphql @@ -800,6 +800,11 @@ input IndexField { field: String } +enum IndexKind { + "ordered scalar index" + ordered +} + "The `Int` scalar type represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1. " scalar Int @@ -1712,6 +1717,26 @@ input NotNullStringOperatorBlock { _nlike: String } +"Configures an ordered index over one or more scalar fields." +input OrderedIndexConfig { + """ + Sets the default index ordering for all fields. + + If a field in the includes list does not specify a direction + the default ordering from this value will be used instead. + """ + direction: Ordering + """ + Sets the fields the index is added on. + + When used on a field definition and the field is not in the includes list + it will be implicitly added as the first entry. + """ + includes: [IndexField] + "Makes the index unique." + unique: Boolean +} + enum Ordering { """ @@ -2552,6 +2577,21 @@ enum VectorDistanceMetric { EUCLIDEAN } +enum VectorIndexAlgorithm { + "HNSW (Hierarchical Navigable Small World)" + hnsw +} + +"Configures an approximate-nearest-neighbour index over a vector field." +input VectorIndexConfig { + "Selects the vector index algorithm using its default configuration." + alg: VectorIndexAlgorithm + "Vector dimensions; must be greater than zero." + dimensions: Int + "Configures the HNSW algorithm." + hnsw: HNSWIndexConfig +} + """ A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document. @@ -2856,7 +2896,7 @@ directive @explain( directive @index( """ Sets the default index ordering for all fields. - + If a field in the includes list does not specify a direction the default ordering from this value will be used instead. """ @@ -2868,10 +2908,16 @@ directive @index( it will be implicitly added as the first entry. """ includes: [IndexField] + "Selects an index kind using its default configuration." + kind: IndexKind "Sets the index name." name: String + "Configures an ordered index." + ordered: OrderedIndexConfig "Makes the index unique." unique: Boolean + "Configures a vector index." + vector: VectorIndexConfig ) on OBJECT | FIELD_DEFINITION """ @@ -2909,12 +2955,4 @@ directive @relation( """ name: String -) on FIELD_DEFINITION - -"@vectorIndex builds an approximate-nearest-neighbour index over a vector field." -directive @vectorIndex( - "Build the index with the HNSW algorithm using these parameters." - HNSW: HNSWIndexConfig - "Vector dimensions; required unless inferable from an @embedding." - dimensions: Int ) on FIELD_DEFINITION \ No newline at end of file diff --git a/internal/request/graphql/schema/types/types.go b/internal/request/graphql/schema/types/types.go index 93c059966f..5ee848fa13 100644 --- a/internal/request/graphql/schema/types/types.go +++ b/internal/request/graphql/schema/types/types.go @@ -49,20 +49,23 @@ const ( IndexDirectivePropUnique = "unique" IndexDirectivePropDirection = "direction" IndexDirectivePropIncludes = "includes" + IndexDirectivePropKind = "kind" + + OrderedIndexKind = "ordered" EncryptedIndexDirectiveLabel = "encryptedIndex" EncryptedIndexDirectivePropType = "type" - VectorIndexDirectiveLabel = "vectorIndex" - VectorIndexDirectivePropDimensions = "dimensions" - VectorIndexDirectivePropHNSW = "HNSW" + VectorIndexKind = "vector" + VectorIndexPropDimensions = "dimensions" + VectorIndexPropAlgorithm = "alg" + VectorIndexPropHNSW = "hnsw" + VectorIndexAlgorithmHNSW = "hnsw" + VectorIndexConfigPropMetric = "metric" - // Fields of the per-algorithm config objects. `metric` is shared in name but each config declares - // its own so a client only sees the knobs of the algorithm it is configuring. - VectorIndexConfigPropMetric = "metric" - VectorIndexConfigPropM = "M" - VectorIndexConfigPropEfConstruction = "efConstruction" - VectorIndexConfigPropEfSearch = "efSearch" + VectorIndexHNSWConfigPropM = "M" + VectorIndexHNSWConfigPropEfConstruction = "efConstruction" + VectorIndexHNSWConfigPropEfSearch = "efSearch" // Values of the VectorDistanceMetric enum. They are the string form of the matching // [client.DistanceMetric], so the two cannot drift apart and the directive parser maps one to the @@ -106,7 +109,32 @@ func OrderingEnum() *gql.Enum { }) } -// VectorDistanceMetricEnum is an enum for the `metric` field of a @vectorIndex algorithm config. +func IndexKindEnum() *gql.Enum { + return gql.NewEnum(gql.EnumConfig{ + Name: "IndexKind", + Values: gql.EnumValueConfigMap{ + OrderedIndexKind: &gql.EnumValueConfig{ + Description: "ordered scalar index", + Value: OrderedIndexKind, + }, + }, + }) +} + +// VectorIndexAlgorithmEnum identifies the configured vector index algorithm. +func VectorIndexAlgorithmEnum() *gql.Enum { + return gql.NewEnum(gql.EnumConfig{ + Name: "VectorIndexAlgorithm", + Values: gql.EnumValueConfigMap{ + VectorIndexAlgorithmHNSW: &gql.EnumValueConfig{ + Description: "HNSW (Hierarchical Navigable Small World)", + Value: VectorIndexAlgorithmHNSW, + }, + }, + }) +} + +// VectorDistanceMetricEnum is an enum for a vector index algorithm's metric field. func VectorDistanceMetricEnum() *gql.Enum { return gql.NewEnum(gql.EnumConfig{ Name: "VectorDistanceMetric", @@ -128,9 +156,7 @@ func VectorDistanceMetricEnum() *gql.Enum { }) } -// HNSWIndexConfigInputObject is the typed config for @vectorIndex's `HNSW` argument. Keying the -// params under the algorithm keeps them type-safe and off the directive's top-level namespace, so a -// client configuring HNSW only sees HNSW's knobs (and later, IVFFlat only sees IVFFlat's). +// HNSWIndexConfigInputObject is the typed config for a vector index's HNSW argument. func HNSWIndexConfigInputObject(metricEnum *gql.Enum) *gql.InputObject { return gql.NewInputObject(gql.InputObjectConfig{ Name: "HNSWIndexConfig", @@ -141,17 +167,17 @@ func HNSWIndexConfigInputObject(metricEnum *gql.Enum) *gql.InputObject { Type: metricEnum, DefaultValue: VectorDistanceMetricCosine, }, - VectorIndexConfigPropM: &gql.InputObjectFieldConfig{ + VectorIndexHNSWConfigPropM: &gql.InputObjectFieldConfig{ Description: "Max connections per node. Higher improves recall at the cost of memory and build time.", Type: gql.Int, DefaultValue: int(client.DefaultHNSWM), }, - VectorIndexConfigPropEfConstruction: &gql.InputObjectFieldConfig{ + VectorIndexHNSWConfigPropEfConstruction: &gql.InputObjectFieldConfig{ Description: "Build-time exploration factor. Higher improves graph quality (recall) at the cost of build time.", Type: gql.Int, DefaultValue: int(client.DefaultHNSWEfConstruction), }, - VectorIndexConfigPropEfSearch: &gql.InputObjectFieldConfig{ + VectorIndexHNSWConfigPropEfSearch: &gql.InputObjectFieldConfig{ Description: "Default query-time exploration factor. Higher improves recall at the cost of query latency.", Type: gql.Int, DefaultValue: int(client.DefaultHNSWEfSearch), @@ -187,7 +213,7 @@ func DefaultDirective() *gql.Directive { return gql.NewDirective(gql.DirectiveConfig{ Name: DefaultDirectiveLabel, Description: `@default is a directive that can be used to set a default field value. - + Setting a default value on a field within a view has no effect.`, Args: gql.FieldConfigArgument{ DefaultDirectivePropValue: &gql.ArgumentConfig{ @@ -266,7 +292,13 @@ func IndexFieldInputObject(orderingEnum *gql.Enum) *gql.InputObject { }) } -func IndexDirective(orderingEnum *gql.Enum, indexFieldInputObject *gql.InputObject) *gql.Directive { +func IndexDirective( + orderingEnum *gql.Enum, + indexFieldInputObject *gql.InputObject, + indexKindEnum *gql.Enum, + orderedIndexInputObject *gql.InputObject, + vectorIndexInputObject *gql.InputObject, +) *gql.Directive { return gql.NewDirective(gql.DirectiveConfig{ Name: IndexDirectiveLabel, Description: "@index is a directive that can be used to add an index on a type or a field.", @@ -275,13 +307,31 @@ func IndexDirective(orderingEnum *gql.Enum, indexFieldInputObject *gql.InputObje Description: "Sets the index name.", Type: gql.String, }, + + IndexDirectivePropKind: &gql.ArgumentConfig{ + Description: "Selects an index kind using its default configuration.", + Type: indexKindEnum, + }, + + VectorIndexKind: &gql.ArgumentConfig{ + Description: "Configures a vector index.", + Type: vectorIndexInputObject, + }, + + OrderedIndexKind: &gql.ArgumentConfig{ + Description: "Configures an ordered index.", + Type: orderedIndexInputObject, + }, + + // unique, direction, and includes are kept here at the top level for backwards compat. + // They are replicated in the `OrderedIndexInputObject` IndexDirectivePropUnique: &gql.ArgumentConfig{ Description: "Makes the index unique.", Type: gql.Boolean, }, IndexDirectivePropDirection: &gql.ArgumentConfig{ Description: `Sets the default index ordering for all fields. - + If a field in the includes list does not specify a direction the default ordering from this value will be used instead.`, Type: orderingEnum, @@ -357,7 +407,7 @@ func CRDTEnum() *gql.Enum { client.PN_COUNTER.String(): &gql.EnumValueConfig{ Value: client.PN_COUNTER, Description: `Positive-Negative Counter. - + WARNING: Incrementing an integer and causing it to overflow the int64 max value will cause the value to roll over to the int64 min value. Incremeting a float and causing it to overflow the float64 max value will act like a no-op.`, @@ -365,7 +415,7 @@ func CRDTEnum() *gql.Enum { client.P_COUNTER.String(): &gql.EnumValueConfig{ Value: client.P_COUNTER, Description: `Positive Counter. - + WARNING: Incrementing an integer and causing it to overflow the int64 max value will cause the value to roll over to the int64 min value. Incremeting a float and causing it to overflow the float64 max value will act like a no-op.`, @@ -458,26 +508,55 @@ func EncryptedIndexDirective() *gql.Directive { }) } -// VectorIndexDirective @vectorIndex builds an approximate-nearest-neighbour index over a vector -// field. The algorithm is chosen by which config argument is set (HNSW today), and its parameters -// live in that config object. `dimensions` is top-level because it describes the vector field, not -// the algorithm. Omitting the algorithm config indexes with HNSW defaults. -func VectorIndexDirective(hnswConfig *gql.InputObject) *gql.Directive { - return gql.NewDirective(gql.DirectiveConfig{ - Name: VectorIndexDirectiveLabel, - Description: "@vectorIndex builds an approximate-nearest-neighbour index over a vector field.", - Args: gql.FieldConfigArgument{ - VectorIndexDirectivePropDimensions: &gql.ArgumentConfig{ - Description: "Vector dimensions; required unless inferable from an @embedding.", +// VectorIndexInputObject configures an approximate-nearest-neighbour index over a vector field. +func VectorIndexInputObject( + hnswConfig *gql.InputObject, + algorithmEnum *gql.Enum, +) *gql.InputObject { + return gql.NewInputObject(gql.InputObjectConfig{ + Name: "VectorIndexConfig", + Description: "Configures an approximate-nearest-neighbour index over a vector field.", + Fields: gql.InputObjectConfigFieldMap{ + VectorIndexPropDimensions: &gql.InputObjectFieldConfig{ + Description: "Vector dimensions; must be greater than zero.", Type: gql.Int, }, - VectorIndexDirectivePropHNSW: &gql.ArgumentConfig{ - Description: "Build the index with the HNSW algorithm using these parameters.", + VectorIndexPropAlgorithm: &gql.InputObjectFieldConfig{ + Description: "Selects the vector index algorithm using its default configuration.", + Type: algorithmEnum, + }, + VectorIndexPropHNSW: &gql.InputObjectFieldConfig{ + Description: "Configures the HNSW algorithm.", Type: hnswConfig, }, }, - Locations: []string{ - gql.DirectiveLocationFieldDefinition, + }) +} + +// OrderedIndexInputObject configures an ordered index over one or more scalar fields. +func OrderedIndexInputObject(orderingEnum *gql.Enum, indexFieldInputObject *gql.InputObject) *gql.InputObject { + return gql.NewInputObject(gql.InputObjectConfig{ + Name: "OrderedIndexConfig", + Description: "Configures an ordered index over one or more scalar fields.", + Fields: gql.InputObjectConfigFieldMap{ + IndexDirectivePropUnique: &gql.InputObjectFieldConfig{ + Description: "Makes the index unique.", + Type: gql.Boolean, + }, + IndexDirectivePropDirection: &gql.InputObjectFieldConfig{ + Description: `Sets the default index ordering for all fields. + + If a field in the includes list does not specify a direction + the default ordering from this value will be used instead.`, + Type: orderingEnum, + }, + IndexDirectivePropIncludes: &gql.InputObjectFieldConfig{ + Description: `Sets the fields the index is added on. + + When used on a field definition and the field is not in the includes list + it will be implicitly added as the first entry.`, + Type: gql.NewList(indexFieldInputObject), + }, }, }) } diff --git a/internal/request/graphql/schema/vector_index_parse_test.go b/internal/request/graphql/schema/vector_index_parse_test.go index 0cc10c3c1d..917b6311d6 100644 --- a/internal/request/graphql/schema/vector_index_parse_test.go +++ b/internal/request/graphql/schema/vector_index_parse_test.go @@ -22,9 +22,9 @@ import ( func TestParseVectorIndex_OnField_ParsesArgsAndDefaults(t *testing.T) { cases := []indexTestCase{ { - description: "vector index with explicit args", + description: "alg selector uses the default HNSW configuration", sdl: `type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + embedding: [Float32!] @index(vector: {dimensions: 3, alg: hnsw}) }`, targetDescriptions: []client.NewIndexRequest{ { @@ -45,18 +45,22 @@ func TestParseVectorIndex_OnField_ParsesArgsAndDefaults(t *testing.T) { }, }, { - description: "vector index with custom HNSW params", + description: "matching alg and HNSW config are allowed", sdl: `type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {M: 32, efConstruction: 200, efSearch: 100}) + embedding: [Float32!] @index( + name: "embeddingIndex", + vector: {dimensions: 3, alg: hnsw, hnsw: {metric: EUCLIDEAN, M: 32, efConstruction: 200, efSearch: 100}} + ) }`, targetDescriptions: []client.NewIndexRequest{ { + Name: "embeddingIndex", Fields: []client.IndexedFieldDescription{ {Name: "embedding"}, }, Vector: &client.VectorIndexDescription{ Algorithm: client.VectorAlgorithmHNSW, - Metric: client.DistanceMetricCosine, + Metric: client.DistanceMetricEuclidean, Dimensions: 3, HNSW: &client.HNSWParams{ M: 32, @@ -79,7 +83,7 @@ func TestParseVectorIndex_OnField_ProducesVectorKindIndex(t *testing.T) { require.NoError(t, err) parseResult, err := schemaManager.ParseSDL(`type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`) require.NoError(t, err) require.Len(t, parseResult, 1) @@ -109,33 +113,68 @@ func TestParseVectorIndex_OnField_ProducesVectorKindIndex(t *testing.T) { func TestParseVectorIndex_InvalidArgs_ReturnsError(t *testing.T) { cases := []invalidIndexTestCase{ { - description: "unknown algorithm is an unknown argument (algorithm is the argument key)", + description: "vector is not an index kind", + sdl: `type user { + embedding: [Float32!] @index(kind: vector) + }`, + expectedErr: `Expected type "IndexKind", found vector`, + }, + { + description: "unknown algorithm enum", sdl: `type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, IVFFlat: {}) + embedding: [Float32!] @index(vector: {dimensions: 3, alg: IVFFlat}) }`, - expectedErr: `Unknown argument "IVFFlat" on directive "@vectorIndex".`, + expectedErr: `Expected type "VectorIndexAlgorithm", found IVFFlat`, + }, + { + description: "unknown algorithm config field", + sdl: `type user { + embedding: [Float32!] @index(vector: {dimensions: 3, IVFFlat: {}}) + }`, + expectedErr: `In field "IVFFlat": Unknown field.`, }, { description: "unsupported metric inside the HNSW config", sdl: `type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: MANHATTAN}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: MANHATTAN}}) }`, expectedErr: `Expected type "VectorDistanceMetric", found MANHATTAN`, }, { description: "unknown top-level argument", sdl: `type user { - embedding: [Float32!] @vectorIndex(unknown: "something", dimensions: 3) + embedding: [Float32!] @index(unknown: "something", vector: {dimensions: 3}) }`, - expectedErr: `Unknown argument "unknown" on directive "@vectorIndex".`, + expectedErr: `Unknown argument "unknown" on directive "@index".`, + }, + { + description: "unknown field inside the vector config", + sdl: `type user { + embedding: [Float32!] @index(vector: {dimensions: 3, unknown: 1}) + }`, + expectedErr: `In field "unknown": Unknown field.`, }, { description: "unknown field inside the HNSW config", sdl: `type user { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {unknown: 1}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {unknown: 1}}) }`, expectedErr: `In field "unknown": Unknown field.`, }, + { + description: "vector and legacy ordered configs are competing kind selectors", + sdl: `type user { + embedding: [Float32!] @index(vector: {}, unique: false) + }`, + expectedErr: errIndexInvalidArgument, + }, + { + description: "vector config is invalid on an object directive", + sdl: `type user @index(vector: {dimensions: 3, alg: hnsw}) { + embedding: [Float32!] + }`, + expectedErr: errIndexInvalidArgument, + }, } for _, test := range cases { diff --git a/tests/integration/collection_version/vector_index_test.go b/tests/integration/collection_version/vector_index_test.go index b3b6db9e21..b358b999b6 100644 --- a/tests/integration/collection_version/vector_index_test.go +++ b/tests/integration/collection_version/vector_index_test.go @@ -19,7 +19,7 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -// A collection carrying a valid @vectorIndex on a raw [Float32!] field is created end-to-end: the +// A collection carrying a valid @index(vector: {...}) on a raw [Float32!] field is created end-to-end: the // index is registered as a vector-kind index with its parsed algorithm/metric/dimensions/HNSW // params. The index performs no graph work yet (Phase 3 wires the HNSW engine); this asserts the // schema surface + descriptor plumbing only. @@ -29,7 +29,7 @@ func TestCollectionVersion_VectorIndexOnRawFloat32Array_ShouldSucceed(t *testing &action.AddCollection{ SDL: ` type Users { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) } `, }, @@ -64,10 +64,10 @@ func TestCollectionVersion_VectorIndexOnFloat32ArrayWithoutDimensionsOrEmbedding &action.AddCollection{ SDL: ` type Users { - embedding: [Float32!] @vectorIndex + embedding: [Float32!] @index(vector: {}) } `, - ExpectedError: "vector index requires dimensions unless field is an embedding", + ExpectedError: "vector index dimensions must be greater than zero", }, }, } @@ -81,7 +81,7 @@ func TestCollectionVersion_VectorIndexOnStringField_ShouldError(t *testing.T) { &action.AddCollection{ SDL: ` type Users { - embedding: String @vectorIndex(dimensions: 3) + embedding: String @index(vector: {dimensions: 3}) } `, ExpectedError: "unsupported field type for vector index", @@ -98,7 +98,7 @@ func TestCollectionVersion_VectorIndexOnFloat64ArrayField_ShouldError(t *testing &action.AddCollection{ SDL: ` type Users { - embedding: [Float64!] @vectorIndex(dimensions: 3) + embedding: [Float64!] @index(vector: {dimensions: 3}) } `, ExpectedError: "unsupported field type for vector index", @@ -110,17 +110,17 @@ func TestCollectionVersion_VectorIndexOnFloat64ArrayField_ShouldError(t *testing } func TestCollectionVersion_VectorIndexWithUnsupportedAlgorithm_ShouldError(t *testing.T) { - // An unknown algorithm is now an unknown directive argument (the algorithm is the argument key), - // so GraphQL rejects it before the parser runs. + // An unknown algorithm config is an unknown field in the vector config, so GraphQL rejects it + // before the parser runs. test := testUtils.TestCase{ Actions: []any{ &action.AddCollection{ SDL: ` type Users { - embedding: [Float32!] @vectorIndex(dimensions: 3, IVFFlat: {}) + embedding: [Float32!] @index(vector: {dimensions: 3, IVFFlat: {}}) } `, - ExpectedError: `Unknown argument "IVFFlat" on directive "@vectorIndex"`, + ExpectedError: `In field "IVFFlat": Unknown field.`, }, }, } @@ -144,7 +144,7 @@ func vectorIndexMetricTest(sdlMetric string, expected client.DistanceMetric) tes &action.AddCollection{ SDL: ` type Users { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: ` + sdlMetric + `}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: ` + sdlMetric + `}}) } `, }, @@ -175,7 +175,7 @@ func TestCollectionVersion_VectorIndexWithUnsupportedMetric_ShouldError(t *testi &action.AddCollection{ SDL: ` type Users { - embedding: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: MANHATTAN}) + embedding: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: MANHATTAN}}) } `, ExpectedError: `Expected type "VectorDistanceMetric", found MANHATTAN`, diff --git a/tests/integration/index/new_composite_test.go b/tests/integration/index/new_composite_test.go index e8f9b1f699..9a71b5351d 100644 --- a/tests/integration/index/new_composite_test.go +++ b/tests/integration/index/new_composite_test.go @@ -79,7 +79,7 @@ func TestCompositeIndexNew_UsingObjectDirective_SetsDefaultDirection(t *testing. Actions: []any{ &action.AddCollection{ SDL: ` - type User @index(direction: DESC, includes: [{field: "name"}, {field: "age"}]) { + type User @index(ordered: {direction: DESC, includes: [{field: "name"}, {field: "age"}]}) { name: String age: Int } diff --git a/tests/integration/index/patch_test.go b/tests/integration/index/patch_test.go index a2336f0451..8186e3bbe7 100644 --- a/tests/integration/index/patch_test.go +++ b/tests/integration/index/patch_test.go @@ -134,7 +134,7 @@ func TestPatchCollection_ModifyVectorIndexMetric_ShouldError(t *testing.T) { SDL: ` type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) } `, }, diff --git a/tests/integration/index/vector_metrics_test.go b/tests/integration/index/vector_metrics_test.go index 057a1f6870..24937b64fc 100644 --- a/tests/integration/index/vector_metrics_test.go +++ b/tests/integration/index/vector_metrics_test.go @@ -44,8 +44,8 @@ func TestVectorIndex_QueryOnAnyMetric_ShouldUseIndexAndScoreByItsMetric(t *testi &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: ` + - testCase.sdlMetric + `}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: ` + + testCase.sdlMetric + `}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}}, @@ -141,8 +141,8 @@ func TestVectorIndex_SameQueryUsingIndexAndFullScan_ReturnsSameResults(t *testin &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: ` + - testCase.sdlMetric + `}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: ` + + testCase.sdlMetric + `}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "short", "vector": vectors["short"]}}, @@ -177,7 +177,7 @@ func TestVectorIndex_SecondIndexOnFieldWithDifferentMetric_IsRejected(t *testing &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.NewIndex{ @@ -204,7 +204,7 @@ func TestVectorIndex_DropThenRecreateWithDifferentMetric_IsAllowed(t *testing.T) &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}}, diff --git a/tests/integration/index/vector_p2p_test.go b/tests/integration/index/vector_p2p_test.go index 2bfd5fd990..2463d7d64a 100644 --- a/tests/integration/index/vector_p2p_test.go +++ b/tests/integration/index/vector_p2p_test.go @@ -22,7 +22,7 @@ import ( // A document written on one peer and synced to another is added to the replica's graph, so a // similarity query on the replica finds it. This proves the P2P merge maintains the vector index, -// not just direct writes. The @vectorIndex is in the schema so both peers build it the same way. +// not just direct writes. The vector @index directive is in the schema so both peers build it the same way. func TestVectorIndexP2P_ReplicatedDoc_IsSearchableOnReplica(t *testing.T) { test := testUtils.TestCase{ Actions: []any{ @@ -31,7 +31,7 @@ func TestVectorIndexP2P_ReplicatedDoc_IsSearchableOnReplica(t *testing.T) { &action.AddCollection{ SDL: `type Users { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, testUtils.ConnectPeers{ diff --git a/tests/integration/index/vector_params_test.go b/tests/integration/index/vector_params_test.go index ff065fc49a..d180a74df8 100644 --- a/tests/integration/index/vector_params_test.go +++ b/tests/integration/index/vector_params_test.go @@ -27,7 +27,7 @@ func TestVectorIndex_CreateWithOversizedM_IsRejected(t *testing.T) { &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE, M: 100000}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE, M: 100000}}) }`, ExpectedError: "vector index parameter is out of range", }, diff --git a/tests/integration/query/simple/with_similarity_vector_index_test.go b/tests/integration/query/simple/with_similarity_vector_index_test.go index 8bbaf42de1..09b1f388a9 100644 --- a/tests/integration/query/simple/with_similarity_vector_index_test.go +++ b/tests/integration/query/simple/with_similarity_vector_index_test.go @@ -18,7 +18,7 @@ import ( testUtils "github.com/sourcenetwork/defradb/tests/integration" ) -// A _similarity + order DESC + limit query on a ready @vectorIndex returns the k nearest documents +// A _similarity + order DESC + limit query on a ready @index(vector: {...}) returns the k nearest documents // (nearest to [1,0,0] is "x", then "xy") and reads only those k, not the whole collection. The // explain variant asserts two doc fetches (a full scan would read four). func TestQuerySimple_WithSimilarityOnVectorIndex_ReturnsKNearest(t *testing.T) { @@ -27,7 +27,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_ReturnsKNearest(t *testing.T) { &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}}, @@ -72,7 +72,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_ReflectsUpdatedVector(t *testin &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, // a sits off the query axis; b starts even further off. After the update b lands exactly on @@ -113,7 +113,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_ExcludesDeletedDoc(t *testing.T &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, // x is nearest to the query [1,0,0], xy is second nearest. @@ -151,7 +151,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_IsMagnitudeInvariant(t *testing &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "unit", "vector": []float32{1, 0, 0}}}, @@ -193,7 +193,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_AscendingOrderFullScans(t *test &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, }, append(docs, @@ -235,7 +235,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_RespectsOffset(t *testing.T) { &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}}, @@ -270,7 +270,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_WrongLengthQueryErrors(t *testi &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}}, @@ -299,7 +299,7 @@ func TestQuerySimple_WithSimilarityOnVectorIndex_NoOrderDoesNotUseIndex(t *testi &action.AddCollection{ SDL: `type User { name: String - vector: [Float32!] @vectorIndex(dimensions: 3, HNSW: {metric: COSINE}) + vector: [Float32!] @index(vector: {dimensions: 3, hnsw: {metric: COSINE}}) }`, }, &action.AddDoc{DocMap: map[string]any{"name": "x", "vector": []float32{1, 0, 0}}},