diff --git a/.ci/integration.cloudbuild.yaml b/.ci/integration.cloudbuild.yaml index d0d0c285fff..6fafa20d333 100644 --- a/.ci/integration.cloudbuild.yaml +++ b/.ci/integration.cloudbuild.yaml @@ -1356,7 +1356,7 @@ steps: - "SINGLESTORE_USER=$_SINGLESTORE_USER" - "SINGLESTORE_DATABASE=$_SINGLESTORE_DATABASE" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["SINGLESTORE_PASSWORD", "SINGLESTORE_HOST", "CLIENT_ID"] + secretEnv: ["SINGLESTORE_PASSWORD", "SINGLESTORE_HOST", "CLIENT_ID", "API_KEY"] volumes: - name: "go" path: "/gopath" @@ -1370,7 +1370,9 @@ steps: .ci/test_with_coverage.sh \ "SingleStore" \ singlestore \ - singlestore + singlestore \ + "" \ + "API_KEY" else echo "No relevant changes for SingleStore. Skipping shard." exit 0 diff --git a/docs/en/integrations/singlestore/tools/singlestore-sql.md b/docs/en/integrations/singlestore/tools/singlestore-sql.md index 3cf5e394f65..02abb38cd1e 100644 --- a/docs/en/integrations/singlestore/tools/singlestore-sql.md +++ b/docs/en/integrations/singlestore/tools/singlestore-sql.md @@ -91,6 +91,75 @@ templateParameters: description: Table to select from ``` +### Example with Vector Search + +SingleStore supports vector operations. When using an `embeddingModel` with a `singlestore-sql` tool, the tool automatically converts text parameters into a JSON string array. You can then use SingleStore's `JSON_ARRAY_PACK()` function in your SQL statement to pack this string into a binary vector format (BLOB) for vector storage and similarity search. + +#### Define the Embedding Model +See [EmbeddingModels](../../../documentation/configuration/embedding-models/_index.md) for more information. + +```yaml +kind: embeddingModel +name: gemini-model +type: gemini +model: gemini-embedding-001 +apiKey: ${GOOGLE_API_KEY} +dimension: 768 +``` + +#### Vector Ingestion Tool +This tool stores both the raw text and its vector representation. It uses `valueFromParam` to hide the vector conversion logic from the LLM, ensuring the Agent only has to provide the content once. + +```yaml +kind: tool +name: insert_doc_singlestore +type: singlestore-sql +source: my-s2-source +statement: | + INSERT INTO vector_table (id, content, embedding) + VALUES (1, ?, JSON_ARRAY_PACK(?)) +description: | + Index new documents for semantic search in SingleStore. +parameters: + - name: content + type: string + description: The text content to store. + - name: text_to_embed + type: string + # Automatically copies 'content' and converts it to a vector string array + valueFromParam: content + embeddedBy: gemini-model +``` + +#### Vector Search Tool +This tool allows the Agent to perform a natural language search. The query string provided by the Agent is converted into a vector string array before the SQL is executed. + +```yaml +kind: tool +name: search_docs_singlestore +type: singlestore-sql +source: my-s2-source +statement: | + SELECT + id, + content, + DOT_PRODUCT(embedding, JSON_ARRAY_PACK(?)) AS score + FROM + vector_table + ORDER BY + score DESC + LIMIT 1 +description: | + Search for documents in SingleStore using natural language. + Returns the most semantically similar result. +parameters: + - name: query + type: string + description: The search query to be converted to a vector. + embeddedBy: gemini-model +``` + + ## Reference | **field** | **type** | **required** | **description** | diff --git a/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go b/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go index 64ae977e5d3..e65a351e14d 100644 --- a/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go +++ b/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go @@ -128,7 +128,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } func (t Tool) EmbedParams(ctx context.Context, paramValues parameters.ParamValues, embeddingModelsMap map[string]embeddingmodels.EmbeddingModel) (parameters.ParamValues, error) { - return parameters.EmbedParams(ctx, t.Parameters, paramValues, embeddingModelsMap, nil) + return parameters.EmbedParams(ctx, t.Parameters, paramValues, embeddingModelsMap, embeddingmodels.FormatVectorForPgvector) } func (t Tool) Manifest() tools.Manifest { diff --git a/internal/tools/singlestore/singlestoresql/singlestoresql.go b/internal/tools/singlestore/singlestoresql/singlestoresql.go index e02118bc6df..d65814b4dbb 100644 --- a/internal/tools/singlestore/singlestoresql/singlestoresql.go +++ b/internal/tools/singlestore/singlestoresql/singlestoresql.go @@ -156,7 +156,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } func (t Tool) EmbedParams(ctx context.Context, paramValues parameters.ParamValues, embeddingModelsMap map[string]embeddingmodels.EmbeddingModel) (parameters.ParamValues, error) { - return parameters.EmbedParams(ctx, t.AllParams, paramValues, embeddingModelsMap, nil) + return parameters.EmbedParams(ctx, t.AllParams, paramValues, embeddingModelsMap, embeddingmodels.FormatVectorForPgvector) } func (t Tool) Manifest() tools.Manifest { diff --git a/tests/singlestore/singlestore_integration_test.go b/tests/singlestore/singlestore_integration_test.go index f88c89c15ea..d0aecfdf0d8 100644 --- a/tests/singlestore/singlestore_integration_test.go +++ b/tests/singlestore/singlestore_integration_test.go @@ -263,6 +263,9 @@ func TestSingleStoreToolEndpoints(t *testing.T) { tmplSelectCombined, tmplSelectFilterCombined := getSingleStoreTmplToolStatement() toolsFile = tests.AddTemplateParamConfig(t, toolsFile, SingleStoreToolType, tmplSelectCombined, tmplSelectFilterCombined, "") + insertStmt := `INSERT INTO senseai_docs (content, embedding) VALUES (?, JSON_ARRAY_PACK(?))` + searchStmt := `SELECT content FROM senseai_docs ORDER BY DOT_PRODUCT(embedding, JSON_ARRAY_PACK(?)) DESC LIMIT 1` + toolsFile = tests.AddSemanticSearchConfig(t, toolsFile, SingleStoreToolType, insertStmt, searchStmt) cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...) if err != nil { t.Fatalf("command initialization returned an error: %s", err) @@ -286,4 +289,22 @@ func TestSingleStoreToolEndpoints(t *testing.T) { tests.RunMCPToolCallMethod(t, mcpMyFailToolWant, mcpSelect1Want) tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want) tests.RunToolInvokeWithTemplateParameters(t, tableNameTemplateParam) + + // Create table for semantic search + _, err = pool.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS senseai_docs (id INT AUTO_INCREMENT PRIMARY KEY, content TEXT, embedding BLOB);") + if err != nil { + t.Fatalf("unable to create semantic search table: %s", err) + } + defer func() { + _, err = pool.ExecContext(ctx, "DROP TABLE IF EXISTS senseai_docs;") + if err != nil { + t.Logf("Teardown failed: %s", err) + } + }() + + // Semantic search tests + httpSemanticInsertWant := `null` + mcpSemanticInsertWant := `` + semanticSearchWant := `The quick brown fox jumps over the lazy dog` + tests.RunSemanticSearchToolInvokeTest(t, httpSemanticInsertWant, mcpSemanticInsertWant, semanticSearchWant) }