diff --git a/.ci/integration.cloudbuild.yaml b/.ci/integration.cloudbuild.yaml index 733db8e438a2..7a4cb25a24aa 100644 --- a/.ci/integration.cloudbuild.yaml +++ b/.ci/integration.cloudbuild.yaml @@ -15,7 +15,7 @@ steps: - id: "detect-changes" name: "gcr.io/cloud-builders/git" - waitFor: ["-"] + waitFor: ["-"] entrypoint: "bash" args: - -c @@ -98,7 +98,12 @@ steps: - "CLOUD_SQL_POSTGRES_REGION=$_REGION" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" secretEnv: - ["CLOUD_SQL_POSTGRES_USER", "CLOUD_SQL_POSTGRES_PASS", "CLIENT_ID", "API_KEY"] + [ + "CLOUD_SQL_POSTGRES_USER", + "CLOUD_SQL_POSTGRES_PASS", + "CLIENT_ID", + "API_KEY", + ] volumes: - name: "go" path: "/gopath" @@ -106,7 +111,7 @@ steps: - -c - | PATTERN="cloudsqlpg|internal/sources/postgres|tests/postgres|internal/server/|tests/common.go|tests/tool.go|.ci/" - + if grep -qE "$$PATTERN" /workspace/changed_files.txt; then echo "Relevant changes detected. Starting Cloud SQL Postgres tests..." .ci/test_with_coverage.sh \ @@ -137,7 +142,7 @@ steps: - -c - | PATTERN="alloydb|internal/sources/postgres|tests/postgres|internal/server/|tests/common.go|.ci/" - + if grep -qE "$$PATTERN" /workspace/changed_files.txt; then echo "Relevant changes detected. Starting AlloyDB tests..." .ci/test_with_coverage.sh \ @@ -161,7 +166,13 @@ steps: - "ALLOYDB_POSTGRES_DATABASE=$_DATABASE_NAME" - "ALLOYDB_POSTGRES_REGION=$_REGION" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["ALLOYDB_POSTGRES_USER", "ALLOYDB_POSTGRES_PASSWORD", "CLIENT_ID", "API_KEY"] + secretEnv: + [ + "ALLOYDB_POSTGRES_USER", + "ALLOYDB_POSTGRES_PASSWORD", + "CLIENT_ID", + "API_KEY", + ] volumes: - name: "go" path: "/gopath" @@ -508,7 +519,7 @@ steps: - "SPANNER_DATABASE=$_DATABASE_NAME" - "SPANNER_INSTANCE=$_SPANNER_INSTANCE" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["CLIENT_ID"] + secretEnv: ["CLIENT_ID", "API_KEY"] volumes: - name: "go" path: "/gopath" @@ -536,7 +547,7 @@ steps: - "GOPATH=/gopath" - "NEO4J_DATABASE=$_NEO4J_DATABASE" - "NEO4J_URI=$_NEO4J_URI" - secretEnv: ["NEO4J_USER", "NEO4J_PASS"] + secretEnv: ["NEO4J_USER", "NEO4J_PASS", "API_KEY"] volumes: - name: "go" path: "/gopath" @@ -757,8 +768,7 @@ steps: env: - "GOPATH=/gopath" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: - ["CLIENT_ID"] + secretEnv: ["CLIENT_ID"] volumes: - name: "go" path: "/gopath" @@ -1159,7 +1169,7 @@ steps: env: - "GOPATH=/gopath" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["CLIENT_ID", "ELASTICSEARCH_USER", "ELASTICSEARCH_PASS", "ELASTICSEARCH_HOST"] + secretEnv: ["CLIENT_ID", "API_KEY"] volumes: - name: "go" path: "/gopath" @@ -1188,7 +1198,8 @@ steps: - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - "SNOWFLAKE_DATABASE=$_SNOWFLAKE_DATABASE" - "SNOWFLAKE_SCHEMA=$_SNOWFLAKE_SCHEMA" - secretEnv: ["CLIENT_ID", "SNOWFLAKE_USER", "SNOWFLAKE_PASS", "SNOWFLAKE_ACCOUNT"] + secretEnv: + ["CLIENT_ID", "SNOWFLAKE_USER", "SNOWFLAKE_PASS", "SNOWFLAKE_ACCOUNT"] volumes: - name: "go" path: "/gopath" @@ -1215,7 +1226,8 @@ steps: env: - "GOPATH=/gopath" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["CLIENT_ID", "CASSANDRA_USER", "CASSANDRA_PASS", "CASSANDRA_HOST"] + secretEnv: + ["CLIENT_ID", "CASSANDRA_USER", "CASSANDRA_PASS", "CASSANDRA_HOST"] volumes: - name: "go" path: "/gopath" @@ -1237,13 +1249,14 @@ steps: - id: "oracle" name: ghcr.io/oracle/oraclelinux9-instantclient:23 - waitFor: ["install-dependencies","detect-changes"] + waitFor: ["install-dependencies", "detect-changes"] entrypoint: /bin/bash env: - "GOPATH=/gopath" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - "ORACLE_SERVER_NAME=$_ORACLE_SERVER_NAME" - secretEnv: ["CLIENT_ID", "ORACLE_USERNAME", "ORACLE_PASSWORD", "ORACLE_HOST"] + secretEnv: + ["CLIENT_ID", "ORACLE_USERNAME", "ORACLE_PASSWORD", "ORACLE_HOST"] volumes: - name: "go" path: "/gopath" @@ -1337,7 +1350,7 @@ steps: - "SINGLESTORE_USER=$_SINGLESTORE_USER" - "SINGLESTORE_DATABASE=$_SINGLESTORE_DATABASE" - "SERVICE_ACCOUNT_EMAIL=$SERVICE_ACCOUNT_EMAIL" - secretEnv: ["SINGLESTORE_PASSWORD", "SINGLESTORE_HOST", "CLIENT_ID"] + secretEnv: ["SINGLESTORE_PASSWORD", "SINGLESTORE_HOST", "CLIENT_ID", "API_KEY"] volumes: - name: "go" path: "/gopath" @@ -1351,7 +1364,9 @@ steps: .ci/test_with_coverage.sh \ "SingleStore" \ singlestore \ - singlestore + singlestore \ + "" \ + "API_KEY" else echo "No relevant changes for SingleStore. Skipping shard." exit 0 @@ -1384,7 +1399,6 @@ steps: exit 0 fi - availableSecrets: secretManager: # Common secrets @@ -1474,12 +1488,6 @@ availableSecrets: env: YUGABYTEDB_USER - versionName: projects/$PROJECT_ID/secrets/yugabytedb_pass/versions/latest env: YUGABYTEDB_PASS - - versionName: projects/$PROJECT_ID/secrets/elastic_search_host/versions/latest - env: ELASTICSEARCH_HOST - - versionName: projects/$PROJECT_ID/secrets/elastic_search_user/versions/latest - env: ELASTICSEARCH_USER - - versionName: projects/$PROJECT_ID/secrets/elastic_search_pass/versions/latest - env: ELASTICSEARCH_PASS - versionName: projects/$PROJECT_ID/secrets/snowflake_account/versions/latest env: SNOWFLAKE_ACCOUNT - versionName: projects/$PROJECT_ID/secrets/snowflake_user/versions/latest @@ -1573,4 +1581,4 @@ substitutions: _MARIADB_PORT: "3307" _MARIADB_DATABASE: test_database _SNOWFLAKE_DATABASE: "test" - _SNOWFLAKE_SCHEMA: "PUBLIC" \ No newline at end of file + _SNOWFLAKE_SCHEMA: "PUBLIC" diff --git a/.hugo/hugo.cloudflare.toml b/.hugo/hugo.cloudflare.toml index ead01437dbb0..023d5ecc4a91 100644 --- a/.hugo/hugo.cloudflare.toml +++ b/.hugo/hugo.cloudflare.toml @@ -76,6 +76,10 @@ ignoreFiles = ["quickstart/shared", "quickstart/python", "quickstart/js", "quick # Add a new version block here before every release # The order of versions in this file is mirrored into the dropdown +[[params.versions]] + version = "v1.0.0" + url = "https://mcp-toolbox.dev/v1.0.0/" + [[params.versions]] version = "v0.32.0" url = "https://mcp-toolbox.dev/v0.32.0/" diff --git a/.hugo/hugo.toml b/.hugo/hugo.toml index 057a027e0bbf..eca92d8c4db2 100644 --- a/.hugo/hugo.toml +++ b/.hugo/hugo.toml @@ -61,6 +61,10 @@ ignoreFiles = ["quickstart/shared", "quickstart/python", "quickstart/js", "quick # Add a new version block here before every release # The order of versions in this file is mirrored into the dropdown +[[params.versions]] + version = "v1.0.0" + url = "https://mcp-toolbox.dev/v1.0.0/" + [[params.versions]] version = "v0.32.0" url = "https://mcp-toolbox.dev/v0.32.0/" diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a32ed417ea..66b1e92105e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.0.0](https://github.com/googleapis/mcp-toolbox/compare/v0.32.0...v1.0.0) (2026-04-10) + + +> [!IMPORTANT] +> This is the first stable release. Please review the [UPGRADING.md](UPGRADING.md) guide for instructions on migrating from previous beta versions. + + +### ⚠ BREAKING CHANGES + +* **tools/elasticsearch:** add vector search support and remove query passing through param ([#2891](https://github.com/googleapis/mcp-toolbox/issues/2891)) +* **tools/looker:** refactor looker-git-branch tool into 5 separate tools ([#2976](https://github.com/googleapis/mcp-toolbox/issues/2976)) + +### Features + +* **auth:** Support opaque token validation for `generic` authService ([#2944](https://github.com/googleapis/mcp-toolbox/issues/2944)) ([c924701](https://github.com/googleapis/mcp-toolbox/commit/c924701adede95877594423d78b7ae72fe0b9c82)) +* **cloudsqlpg:** Run `SELECT 1` after successful connection attempt ([#2997](https://github.com/googleapis/mcp-toolbox/issues/2997)) ([6ed9700](https://github.com/googleapis/mcp-toolbox/commit/6ed9700e15f08b31e65eb0afa605f4a8ea937e66)) +* **tools/bigquerysql:** Add semantic search support ([#2890](https://github.com/googleapis/mcp-toolbox/issues/2890)) ([862c396](https://github.com/googleapis/mcp-toolbox/commit/862c396cadfa1d95d12cc121312a81035c22cbad)) +* **tools/elasticsearch-execute-esql:** Add Tool to execute arbitrary ES/QL queries ([#3013](https://github.com/googleapis/mcp-toolbox/issues/3013)) ([ae49fb7](https://github.com/googleapis/mcp-toolbox/commit/ae49fb737031d783b6734a0ea35488dd0f4c7ccc)) +* **tools/elasticsearch:** Add vector search support and remove query passing through param ([#2891](https://github.com/googleapis/mcp-toolbox/issues/2891)) ([d44e879](https://github.com/googleapis/mcp-toolbox/commit/d44e879336f6628790e3f1dca2477cb56fe8f080)) +* **tools/looker:** Refactor looker-git-branch tool into 5 separate tools ([#2976](https://github.com/googleapis/mcp-toolbox/issues/2976)) ([b2472d4](https://github.com/googleapis/mcp-toolbox/commit/b2472d4926dacc496fc6956185fb281b5e75f56f)) +* **tools/mysql:** Add list-table-stats-tool to list table statistics in MySQL and Cloud SQL MySQL source. ([#2938](https://github.com/googleapis/mcp-toolbox/issues/2938)) ([dc2c2b4](https://github.com/googleapis/mcp-toolbox/commit/dc2c2b44e512e34d4d3a0b9c63b59374c37c4c4a)) + ## [0.32.0](https://github.com/googleapis/mcp-toolbox/compare/v0.31.0...v0.32.0) (2026-04-08) diff --git a/README.md b/README.md index c5e0ffb9a3e4..718babe15abd 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ To install Toolbox as a binary: > > ```sh > # see releases page for other versions -> export VERSION=0.32.0 +> export VERSION=1.0.0 > curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/linux/amd64/toolbox > chmod +x toolbox > ``` @@ -253,7 +253,7 @@ To install Toolbox as a binary: > > ```sh > # see releases page for other versions -> export VERSION=0.32.0 +> export VERSION=1.0.0 > curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/darwin/arm64/toolbox > chmod +x toolbox > ``` @@ -266,7 +266,7 @@ To install Toolbox as a binary: > > ```sh > # see releases page for other versions -> export VERSION=0.32.0 +> export VERSION=1.0.0 > curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/darwin/amd64/toolbox > chmod +x toolbox > ``` @@ -279,7 +279,7 @@ To install Toolbox as a binary: > > ```cmd > :: see releases page for other versions -> set VERSION=0.32.0 +> set VERSION=1.0.0 > curl -o toolbox.exe "https://storage.googleapis.com/mcp-toolbox-for-databases/v%VERSION%/windows/amd64/toolbox.exe" > ``` > @@ -291,7 +291,7 @@ To install Toolbox as a binary: > > ```powershell > # see releases page for other versions -> $VERSION = "0.32.0" +> $VERSION = "1.0.0" > curl.exe -o toolbox.exe "https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/windows/amd64/toolbox.exe" > ``` > @@ -304,7 +304,7 @@ You can also install Toolbox as a container: ```sh # see releases page for other versions -export VERSION=0.32.0 +export VERSION=1.0.0 docker pull us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:$VERSION ``` @@ -328,7 +328,7 @@ To install from source, ensure you have the latest version of [Go installed](https://go.dev/doc/install), and then run the following command: ```sh -go install github.com/googleapis/mcp-toolbox@v0.32.0 +go install github.com/googleapis/mcp-toolbox@v1.0.0 ``` diff --git a/UPGRADING.md b/UPGRADING.md index f5d9c408b494..2de513562905 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -16,7 +16,16 @@ This guide outlines what has changed and the steps you need to take to upgrade. ## 🚨 Breaking Changes (Action Required) -### 1. Endpoint Transition: `/api` disabled by default +### 1. Repository Rename: genai-toolbox ➡️ mcp-toolbox +The GitHub repository has been officially renamed to `googleapis/mcp-toolbox`. To update your local environment, run the following commands: + +1. Rename your local directory: `cd .. && mv genai-toolbox mcp-toolbox && cd mcp-toolbox` + +2. Update the remote URL: `git remote set-url origin git@github.com:googleapis/mcp-toolbox.git` + +3. Verify the update: `git remote -v` + +### 2. Endpoint Transition: `/api` disabled by default The legacy `/api` endpoint for the native Toolbox protocol is now disabled by default. All official SDKs have been updated to use the `/mcp` endpoint, which aligns with the standard Model Context Protocol (MCP) specification. If you still require the legacy `/api` endpoint, you must explicitly activate it using a new command-line flag. @@ -27,15 +36,15 @@ If you still require the legacy `/api` endpoint, you must explicitly activate it relied on a non-standard feature that is missing from the new implementation, please submit a feature request on our [GitHub Issues page](https://github.com/googleapis/mcp-toolbox/issues). -### 2. Strict Tool Naming Validation (SEP986) +### 3. Strict Tool Naming Validation (SEP986) Tool names are now strictly validated against [ModelContextProtocol SEP986 guidelines](https://github.com/alexhancock/modelcontextprotocol/blob/main/docs/specification/draft/server/tools.mdx#tool-names) prior to MCP initialization. * **Migration:** Ensure all your tool names **only** contain alphanumeric characters, hyphens (`-`), underscores (`_`), and periods (`.`). Any other special characters will cause initialization to fail. -### 3. Removed CLI Flags +### 4. Removed CLI Flags The legacy snake_case flag `--tools_file` has been completely removed. * **Migration:** Update your deployment scripts to use `--config` instead. -### 4. Singular `kind` Values in Configuration +### 5. Singular `kind` Values in Configuration _(This step applies only if you are currently using the new flat format.)_ All primitive kind fields in configuration files have been updated to use singular nouns instead of plural. For example, `kind: sources` is now `kind: source`, and `kind: tools` is now `kind: tool`. @@ -44,12 +53,12 @@ All primitive kind fields in configuration files have been updated to use singul values. _(Note: If you transitioned to the flat format using the `./toolbox migrate` command, this step was handled automatically.)_ -### 5. Configuration Schema: `authSources` renamed +### 6. Configuration Schema: `authSources` renamed The `authSources` field is no longer supported in configuration files. * **Migration:** Rename all instances of `authSources` to `authService` in your configuration files. -### 6. CloudSQL for SQL Server: `ipAddress` removed +### 7. CloudSQL for SQL Server: `ipAddress` removed The `ipAddress` field for the CloudSQL for SQL Server source was redundant and has been removed. * **Migration:** Remove the `ipAddress` field from your CloudSQL for SQL Server configurations. diff --git a/cmd/internal/config_test.go b/cmd/internal/config_test.go index 8985494ed8d5..d9bbeaa53e67 100644 --- a/cmd/internal/config_test.go +++ b/cmd/internal/config_test.go @@ -1763,7 +1763,7 @@ func TestPrebuiltTools(t *testing.T) { }, "monitor": tools.ToolsetConfig{ Name: "monitor", - ToolNames: []string{"get_query_plan", "list_active_queries", "get_query_metrics", "get_system_metrics", "list_table_fragmentation", "list_tables_missing_unique_indexes"}, + ToolNames: []string{"get_query_plan", "list_active_queries", "get_query_metrics", "get_system_metrics", "list_table_fragmentation", "list_table_stats", "list_tables_missing_unique_indexes"}, }, "lifecycle": tools.ToolsetConfig{ Name: "lifecycle", @@ -1847,7 +1847,7 @@ func TestPrebuiltTools(t *testing.T) { }, "monitor": tools.ToolsetConfig{ Name: "monitor", - ToolNames: []string{"get_query_plan", "list_active_queries", "list_table_fragmentation", "list_tables_missing_unique_indexes"}, + ToolNames: []string{"get_query_plan", "list_active_queries", "list_table_fragmentation", "list_table_stats", "list_tables_missing_unique_indexes"}, }, }, }, diff --git a/cmd/internal/imports.go b/cmd/internal/imports.go index e84192909a53..edbf8d0cde29 100644 --- a/cmd/internal/imports.go +++ b/cmd/internal/imports.go @@ -182,6 +182,7 @@ import ( _ "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqllisttablefragmentation" _ "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqllisttables" _ "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqllisttablesmissinguniqueindexes" + _ "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqllisttablestats" _ "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqlsql" _ "github.com/googleapis/mcp-toolbox/internal/tools/neo4j/neo4jcypher" _ "github.com/googleapis/mcp-toolbox/internal/tools/neo4j/neo4jexecutecypher" diff --git a/cmd/version.txt b/cmd/version.txt index 9eb2aa3f1095..3eefcb9dd5b3 100644 --- a/cmd/version.txt +++ b/cmd/version.txt @@ -1 +1 @@ -0.32.0 +1.0.0 diff --git a/docs/CLOUDSQLMYSQL_README.md b/docs/CLOUDSQLMYSQL_README.md index f7f4c1d8c188..55bdd7ab66ef 100644 --- a/docs/CLOUDSQLMYSQL_README.md +++ b/docs/CLOUDSQLMYSQL_README.md @@ -61,6 +61,7 @@ The Cloud SQL for MySQL MCP server provides the following tools: | `list_tables` | Lists detailed schema information for user-created tables. | | `list_tables_missing_unique_indexes` | Find tables that do not have primary or unique key constraint. | | `list_table_fragmentation` | List table fragmentation in MySQL. | +| `list_table_stats` | List table statistics in MySQL. | ## Custom MCP Server Configuration diff --git a/docs/en/documentation/connect-to/ides/looker_mcp.md b/docs/en/documentation/connect-to/ides/looker_mcp.md index 20c2282457bd..c48bf2938d3b 100644 --- a/docs/en/documentation/connect-to/ides/looker_mcp.md +++ b/docs/en/documentation/connect-to/ides/looker_mcp.md @@ -109,19 +109,19 @@ After you install Looker in the MCP Store, resources and tools from the server a {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/mssql_mcp.md b/docs/en/documentation/connect-to/ides/mssql_mcp.md index e18c8e98dacb..d3d298f4c654 100644 --- a/docs/en/documentation/connect-to/ides/mssql_mcp.md +++ b/docs/en/documentation/connect-to/ides/mssql_mcp.md @@ -45,19 +45,19 @@ instance: {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/mysql_mcp.md b/docs/en/documentation/connect-to/ides/mysql_mcp.md index a5775b5edf4b..7cbb1c187a71 100644 --- a/docs/en/documentation/connect-to/ides/mysql_mcp.md +++ b/docs/en/documentation/connect-to/ides/mysql_mcp.md @@ -43,19 +43,19 @@ expose your developer assistant tools to a MySQL instance: {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/neo4j_mcp.md b/docs/en/documentation/connect-to/ides/neo4j_mcp.md index 2b818cb88768..b09e68aa7207 100644 --- a/docs/en/documentation/connect-to/ides/neo4j_mcp.md +++ b/docs/en/documentation/connect-to/ides/neo4j_mcp.md @@ -44,19 +44,19 @@ expose your developer assistant tools to a Neo4j instance: {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/oracle_mcp.md b/docs/en/documentation/connect-to/ides/oracle_mcp.md index 6f09cc3e4faa..e98d03c61f7e 100644 --- a/docs/en/documentation/connect-to/ides/oracle_mcp.md +++ b/docs/en/documentation/connect-to/ides/oracle_mcp.md @@ -46,19 +46,19 @@ to expose your developer assistant tools to an Oracle instance: {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/postgres_mcp.md b/docs/en/documentation/connect-to/ides/postgres_mcp.md index c252aa763e20..40a70d770dec 100644 --- a/docs/en/documentation/connect-to/ides/postgres_mcp.md +++ b/docs/en/documentation/connect-to/ides/postgres_mcp.md @@ -56,19 +56,19 @@ Omni](https://cloud.google.com/alloydb/omni/docs/overview). {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/connect-to/ides/sqlite_mcp.md b/docs/en/documentation/connect-to/ides/sqlite_mcp.md index ec79bc1d037f..d86390033ea8 100644 --- a/docs/en/documentation/connect-to/ides/sqlite_mcp.md +++ b/docs/en/documentation/connect-to/ides/sqlite_mcp.md @@ -43,19 +43,19 @@ to expose your developer assistant tools to a SQLite instance: {{< tabpane persist=header >}} {{< tab header="linux/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/linux/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/linux/amd64/toolbox {{< /tab >}} {{< tab header="darwin/arm64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/arm64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/arm64/toolbox {{< /tab >}} {{< tab header="darwin/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/darwin/amd64/toolbox +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/darwin/amd64/toolbox {{< /tab >}} {{< tab header="windows/amd64" lang="bash" >}} -curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/windows/amd64/toolbox.exe +curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/windows/amd64/toolbox.exe {{< /tab >}} {{< /tabpane >}} diff --git a/docs/en/documentation/getting-started/colab_quickstart.ipynb b/docs/en/documentation/getting-started/colab_quickstart.ipynb index 1e97c433fb06..111e5dae3bd8 100644 --- a/docs/en/documentation/getting-started/colab_quickstart.ipynb +++ b/docs/en/documentation/getting-started/colab_quickstart.ipynb @@ -249,7 +249,7 @@ }, "outputs": [], "source": [ - "version = \"0.32.0\" # x-release-please-version\n", + "version = \"1.0.0\" # x-release-please-version\n", "! curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v{version}/linux/amd64/toolbox\n", "\n", "# Make the binary executable\n", diff --git a/docs/en/documentation/getting-started/mcp_quickstart/_index.md b/docs/en/documentation/getting-started/mcp_quickstart/_index.md index 7f713996533d..72a4b22c05c5 100644 --- a/docs/en/documentation/getting-started/mcp_quickstart/_index.md +++ b/docs/en/documentation/getting-started/mcp_quickstart/_index.md @@ -107,7 +107,7 @@ In this section, we will download Toolbox, configure our tools in a ```bash export OS="linux/amd64" # one of linux/amd64, darwin/arm64, darwin/amd64, or windows/amd64 - curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/$OS/toolbox + curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/$OS/toolbox ``` diff --git a/docs/en/documentation/getting-started/quickstart/shared/configure_toolbox.md b/docs/en/documentation/getting-started/quickstart/shared/configure_toolbox.md index 8158b5c8bce9..62219410b183 100644 --- a/docs/en/documentation/getting-started/quickstart/shared/configure_toolbox.md +++ b/docs/en/documentation/getting-started/quickstart/shared/configure_toolbox.md @@ -13,7 +13,7 @@ In this section, we will download Toolbox, configure our tools in a ```bash export OS="linux/amd64" # one of linux/amd64, darwin/arm64, darwin/amd64, or windows/amd64 - curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v0.32.0/$OS/toolbox + curl -O https://storage.googleapis.com/mcp-toolbox-for-databases/v1.0.0/$OS/toolbox ``` diff --git a/docs/en/documentation/introduction/_index.md b/docs/en/documentation/introduction/_index.md index 554405651a0c..102f8c9b6973 100644 --- a/docs/en/documentation/introduction/_index.md +++ b/docs/en/documentation/introduction/_index.md @@ -110,7 +110,7 @@ To install Toolbox as a binary on Linux (AMD64): ```sh # see releases page for other versions -export VERSION=0.32.0 +export VERSION=1.0.0 curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/linux/amd64/toolbox chmod +x toolbox ``` @@ -121,7 +121,7 @@ To install Toolbox as a binary on macOS (Apple Silicon): ```sh # see releases page for other versions -export VERSION=0.32.0 +export VERSION=1.0.0 curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/darwin/arm64/toolbox chmod +x toolbox ``` @@ -132,7 +132,7 @@ To install Toolbox as a binary on macOS (Intel): ```sh # see releases page for other versions -export VERSION=0.32.0 +export VERSION=1.0.0 curl -L -o toolbox https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/darwin/amd64/toolbox chmod +x toolbox ``` @@ -143,7 +143,7 @@ To install Toolbox as a binary on Windows (Command Prompt): ```cmd :: see releases page for other versions -set VERSION=0.32.0 +set VERSION=1.0.0 curl -o toolbox.exe "https://storage.googleapis.com/mcp-toolbox-for-databases/v%VERSION%/windows/amd64/toolbox.exe" ``` @@ -153,7 +153,7 @@ To install Toolbox as a binary on Windows (PowerShell): ```powershell # see releases page for other versions -$VERSION = "0.32.0" +$VERSION = "1.0.0" curl.exe -o toolbox.exe "https://storage.googleapis.com/mcp-toolbox-for-databases/v$VERSION/windows/amd64/toolbox.exe" ``` @@ -165,7 +165,7 @@ You can also install Toolbox as a container: ```sh # see releases page for other versions -export VERSION=0.32.0 +export VERSION=1.0.0 docker pull us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:$VERSION ``` @@ -184,7 +184,7 @@ To install from source, ensure you have the latest version of [Go installed](https://go.dev/doc/install), and then run the following command: ```sh -go install github.com/googleapis/mcp-toolbox@v0.32.0 +go install github.com/googleapis/mcp-toolbox@v1.0.0 ``` {{% /tab %}} diff --git a/docs/en/integrations/cloud-sql-mysql/prebuilt-configs/cloud-sql-for-mysql.md b/docs/en/integrations/cloud-sql-mysql/prebuilt-configs/cloud-sql-for-mysql.md index 594fb3d33008..80897e02240e 100644 --- a/docs/en/integrations/cloud-sql-mysql/prebuilt-configs/cloud-sql-for-mysql.md +++ b/docs/en/integrations/cloud-sql-mysql/prebuilt-configs/cloud-sql-for-mysql.md @@ -30,3 +30,4 @@ description: "Details of the Cloud SQL for MySQL prebuilt configuration." * `list_tables_missing_unique_indexes`: Looks for tables that do not have primary or unique key contraint. * `list_table_fragmentation`: Displays table fragmentation in MySQL. + * `list_table_stats`: Displays table statistics in MySQL. diff --git a/docs/en/integrations/mysql/prebuilt-configs/mysql.md b/docs/en/integrations/mysql/prebuilt-configs/mysql.md index 85ba36c030e6..643202fdda8b 100644 --- a/docs/en/integrations/mysql/prebuilt-configs/mysql.md +++ b/docs/en/integrations/mysql/prebuilt-configs/mysql.md @@ -25,3 +25,4 @@ description: "Details of the MySQL prebuilt configuration." * `list_tables_missing_unique_indexes`: Looks for tables that do not have primary or unique key contraint. * `list_table_fragmentation`: Displays table fragmentation in MySQL. + * `list_table_stats`: Displays table statistics in MySQL. diff --git a/docs/en/integrations/mysql/tools/mysql-list-table-stats.md b/docs/en/integrations/mysql/tools/mysql-list-table-stats.md new file mode 100644 index 000000000000..a755884e29ce --- /dev/null +++ b/docs/en/integrations/mysql/tools/mysql-list-table-stats.md @@ -0,0 +1,80 @@ +--- +title: "mysql-list-table-stats" +type: docs +weight: 1 +description: > + A "mysql-list-table-stats" tool report table statistics including table size, total latency, rows read, rows written, read and write latency for entire instance, a specified database, or a specified table. +--- + +## About + +A `mysql-list-table-stats` tool generates table-level performance and resource consumption statistics to facilitate bottleneck identification and workload analysis. + +`mysql-list-table-stats` outputs detailed table-level resource consumption including estimated row counts, table size, a complete breakdown of CRUD activity (rows fetched, inserted, updated, and deleted), and IO statistics such as total, read, write and miscellaneous latency. The output is a JSON formatted array of the top 10 MySQL tables ranked by total latency. + +Below are some use cases for `mysql-list-table-stats` +- **Finding hottest tables**: Identify tables with highest total latency, read or writes based on the `sort_by` column. +- **Finding tables with most reads**: Identify tables with highest reads by sorting on `rows_fetched`. +- **Monitoring growth**: Track `row_count` and `size_MB` of table over time to estimate growth." + +## Compatible Sources + +{{< compatible-sources others="integrations/cloud-sql-mysql">}} + +## Requirements + +- `performance_schema` should be turned ON for this tool to work. + +## Parameters + +This tool takes 4 optional input parameters: + +- `table_schema` (optional): The database where table stats check is to be + executed. Check all tables visible to the current database if not specified. +- `table_name` (optional): Name of the table to be checked. Check all tables + visible to the current user if not specified. +- `sort_by` (optional): The column to sort by. Valid values are `row_count`, `rows_fetched`, `rows_inserted`, `rows_updated`, `rows_deleted`, `total_latency_secs` (defaults to `total_latency_secs`) +- `limit` (optional): Max rows to return, default 10. + +## Example + +```yaml +kind: tools +name: list_table_stats +type: mysql-list-table-stats +source: my-mysql-instance +description: Display table statistics including table size, total latency, rows read, rows written, read and write latency for entire instance, a specified database, or a specified table. Specifying a database name or table name filters the output to that specific db or table. Results are limited to 10 by default. +``` + +## Output Format + +The response is a json array with the following fields: +```json +[ + { + "table_schema": "The schema/database this table belongs to", + "table_name": "Name of this table", + "size_MB": "Size of the table data in MB", + "row_count": "Number of rows in the table", + "total_latency_secs": "total latency in secs", + "rows_fetched": "total number of rows fetched", + "rows_inserted": "total number of rows inserted", + "rows_updated": "total number of rows updated", + "rows_deleted": "total number of rows deleted", + "io_reads": "total number of io read requests", + "io_read_latency": "io read latency in seconds", + "io_write_latency": "io write latency in seconds", + "io_misc_latency": "io misc latency in seconds", + } +] +``` + +## Reference + +| **field** | **type** | **required** | **description** | +|-------------|:--------:|:------------:|----------------------------------------------------| +| type | string | true | Must be "mysql-list-table-stats". | +| source | string | true | Name of the source the SQL should execute on. | +| description | string | true | Description of the tool that is passed to the LLM. | + + diff --git a/docs/en/integrations/neo4j/tools/neo4j-cypher.md b/docs/en/integrations/neo4j/tools/neo4j-cypher.md index aef621125550..5504ea6907f2 100644 --- a/docs/en/integrations/neo4j/tools/neo4j-cypher.md +++ b/docs/en/integrations/neo4j/tools/neo4j-cypher.md @@ -67,6 +67,71 @@ parameters: description: 4 digit number starting in 1900 up to the current year ``` +### Vector Search + +Neo4j supports vector similarity search. When using an `embeddingModel` with a `neo4j-cypher` tool, the tool automatically converts text parameters into the vector format required by Neo4j. + +#### Define the Embedding Model + +See [EmbeddingModels](../../../documentation/configuration/embedding-models/_index.md) for more information. + +kind: embeddingModel +name: gemini-model +type: gemini +model: gemini-embedding-001 +apiKey: ${GOOGLE_API_KEY} +dimension: 768 + +#### Vector Ingestion Tool + +This tool stores both the raw text and its vector representation. It uses `valueFromParam` to hide the vector conversion logic from the LLM, ensuring the Agent only has to provide the content once. +```yaml +kind: tool +name: insert_doc_neo4j +type: neo4j-cypher +source: my-neo4j-source +statement: | + CREATE (n:Document {content: $content, embedding: $text_to_embed}) + RETURN 1 as result +description: | + Index new documents for semantic search in Neo4j. +parameters: + - name: content + type: string + description: The text content to store. + - name: text_to_embed + type: string + # Automatically copies 'content' and converts it to a vector array + valueFromParam: content + embeddedBy: gemini-model +``` + +#### Vector Search Tool + +This tool allows the Agent to perform a natural language search. The query string provided by the Agent is converted into a vector before the Cypher statement is executed. + +```yaml +kind: tool +name: search_docs_neo4j +type: neo4j-cypher +source: my-neo4j-source +statement: | + MATCH (n:Document) + WITH n, vector.similarity.cosine(n.embedding, $query) AS score + WHERE score IS NOT NULL + ORDER BY score DESC + LIMIT 1 + RETURN n.content as content +description: | + Search for documents in Neo4j using natural language. + Returns the most semantically similar result. +parameters: + - name: query + type: string + description: The search query to be converted to a vector. + embeddedBy: gemini-model +``` + ## Reference | **field** | **type** | **required** | **description** | diff --git a/docs/en/integrations/singlestore/tools/singlestore-sql.md b/docs/en/integrations/singlestore/tools/singlestore-sql.md index 3cf5e394f651..02abb38cd1e8 100644 --- a/docs/en/integrations/singlestore/tools/singlestore-sql.md +++ b/docs/en/integrations/singlestore/tools/singlestore-sql.md @@ -91,6 +91,75 @@ templateParameters: description: Table to select from ``` +### Example with Vector Search + +SingleStore supports vector operations. When using an `embeddingModel` with a `singlestore-sql` tool, the tool automatically converts text parameters into a JSON string array. You can then use SingleStore's `JSON_ARRAY_PACK()` function in your SQL statement to pack this string into a binary vector format (BLOB) for vector storage and similarity search. + +#### Define the Embedding Model +See [EmbeddingModels](../../../documentation/configuration/embedding-models/_index.md) for more information. + +```yaml +kind: embeddingModel +name: gemini-model +type: gemini +model: gemini-embedding-001 +apiKey: ${GOOGLE_API_KEY} +dimension: 768 +``` + +#### Vector Ingestion Tool +This tool stores both the raw text and its vector representation. It uses `valueFromParam` to hide the vector conversion logic from the LLM, ensuring the Agent only has to provide the content once. + +```yaml +kind: tool +name: insert_doc_singlestore +type: singlestore-sql +source: my-s2-source +statement: | + INSERT INTO vector_table (id, content, embedding) + VALUES (1, ?, JSON_ARRAY_PACK(?)) +description: | + Index new documents for semantic search in SingleStore. +parameters: + - name: content + type: string + description: The text content to store. + - name: text_to_embed + type: string + # Automatically copies 'content' and converts it to a vector string array + valueFromParam: content + embeddedBy: gemini-model +``` + +#### Vector Search Tool +This tool allows the Agent to perform a natural language search. The query string provided by the Agent is converted into a vector string array before the SQL is executed. + +```yaml +kind: tool +name: search_docs_singlestore +type: singlestore-sql +source: my-s2-source +statement: | + SELECT + id, + content, + DOT_PRODUCT(embedding, JSON_ARRAY_PACK(?)) AS score + FROM + vector_table + ORDER BY + score DESC + LIMIT 1 +description: | + Search for documents in SingleStore using natural language. + Returns the most semantically similar result. +parameters: + - name: query + type: string + description: The search query to be converted to a vector. + embeddedBy: gemini-model +``` + + ## Reference | **field** | **type** | **required** | **description** | diff --git a/docs/en/integrations/spanner/tools/spanner-sql.md b/docs/en/integrations/spanner/tools/spanner-sql.md index bf56e98cd662..a8e476f665c8 100644 --- a/docs/en/integrations/spanner/tools/spanner-sql.md +++ b/docs/en/integrations/spanner/tools/spanner-sql.md @@ -127,6 +127,76 @@ parameters: {{< /tab >}} {{< /tabpane >}} + +### Example with Vector Search + +Spanner supports high-performance vector similarity search. When using an `embeddingModel` with a `spanner-sql` tool, the tool automatically converts text parameters into the native ARRAY format required by Spanner. + +#### Define the Embedding Model +See [EmbeddingModels](../../../documentation/configuration/embedding-models/_index.md) for more information. + +```yaml +kind: embeddingModel +name: gemini-model +type: gemini +model: gemini-embedding-001 +apiKey: ${GOOGLE_API_KEY} +dimension: 768 +``` + +#### Vector Ingestion Tool +This tool stores both the raw text and its vector representation. It uses `valueFromParam` to hide the vector conversion logic from the LLM, ensuring the Agent only has to provide the content once. + +```yaml +kind: tool +name: insert_doc_spanner +type: spanner-sql +source: my-spanner-source +statement: | + INSERT INTO vector_table (id, content, embedding) + VALUES (1, @content, @text_to_embed) +description: | + Index new documents for semantic search in Spanner. +parameters: + - name: content + type: string + description: The text content to store. + - name: text_to_embed + type: string + # Automatically copies 'content' and converts it to a FLOAT64 array + valueFromParam: content + embeddedBy: gemini-model +``` + +#### Vector Search Tool +This tool allows the Agent to perform a natural language search. The query string provided by the Agent is converted into a vector before the SQL is executed. + +```yaml +kind: tool +name: search_docs_spanner +type: spanner-sql +source: my-spanner-source +statement: | + SELECT + id, + content, + COSINE_DISTANCE(embedding, @query) AS distance + FROM + vector_table + ORDER BY + distance + LIMIT 1 +description: | + Search for documents in Spanner using natural language. + Returns the most semantically similar result. +parameters: + - name: query + type: string + description: The search query to be converted to a vector. + embeddedBy: gemini-model +``` + + ### Example with Template Parameters > **Note:** This tool allows direct modifications to the SQL statement, diff --git a/gemini-extension.json b/gemini-extension.json index f1f68b53a61d..0240ccdf749a 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "mcp-toolbox-for-databases", - "version": "0.32.0", + "version": "1.0.0", "description": "MCP Toolbox for Databases is an open-source MCP server for more than 30 different datasources.", "contextFileName": "MCP-TOOLBOX-EXTENSION.md" } \ No newline at end of file diff --git a/internal/prebuiltconfigs/tools/cloud-sql-mysql.yaml b/internal/prebuiltconfigs/tools/cloud-sql-mysql.yaml index ebf3680b08f3..81d2ac4df3a5 100644 --- a/internal/prebuiltconfigs/tools/cloud-sql-mysql.yaml +++ b/internal/prebuiltconfigs/tools/cloud-sql-mysql.yaml @@ -44,6 +44,10 @@ tools: kind: mysql-list-tables source: cloud-sql-mysql-source description: "Lists detailed schema information (object type, columns, constraints, indexes, triggers, comment) as JSON for user-created tables (ordinary or partitioned). Filters by a comma-separated list of names. If names are omitted, lists all tables in user schemas." + list_table_stats: + kind: mysql-list-table-stats + source: cloud-sql-mysql-source + description: "Display table statistics including table size, total latency, rows read, rows written, read and write latency for entire instance, a specified database, or a specified table. Specifying a database name or table name filters the output to that specific db or table. Results are limited to 10 by default." list_tables_missing_unique_indexes: kind: mysql-list-tables-missing-unique-indexes source: cloud-sql-mysql-source @@ -148,6 +152,7 @@ toolsets: - get_query_metrics - get_system_metrics - list_table_fragmentation + - list_table_stats - list_tables_missing_unique_indexes lifecycle: - create_backup diff --git a/internal/prebuiltconfigs/tools/mysql.yaml b/internal/prebuiltconfigs/tools/mysql.yaml index ce99d0375771..9fd2989dc2a8 100644 --- a/internal/prebuiltconfigs/tools/mysql.yaml +++ b/internal/prebuiltconfigs/tools/mysql.yaml @@ -43,6 +43,10 @@ tools: kind: mysql-list-tables source: mysql-source description: "Lists detailed schema information (object type, columns, constraints, indexes, triggers, comment) as JSON for user-created tables (ordinary or partitioned). Filters by a comma-separated list of names. If names are omitted, lists all tables in user schemas." + list_table_stats: + kind: mysql-list-table-stats + source: mysql-source + description: "Display table statistics including table size, total latency, rows read, rows written, read and write latency for entire instance, a specified database, or a specified table. Specifying a database name or table name filters the output to that specific db or table. Results are limited to 10 by default." list_tables_missing_unique_indexes: kind: mysql-list-tables-missing-unique-indexes source: mysql-source @@ -61,4 +65,5 @@ toolsets: - get_query_plan - list_active_queries - list_table_fragmentation + - list_table_stats - list_tables_missing_unique_indexes diff --git a/internal/sources/cloudsqlmysql/cloud_sql_mysql.go b/internal/sources/cloudsqlmysql/cloud_sql_mysql.go index 0ce9affaefa1..cce65db37497 100644 --- a/internal/sources/cloudsqlmysql/cloud_sql_mysql.go +++ b/internal/sources/cloudsqlmysql/cloud_sql_mysql.go @@ -102,6 +102,10 @@ func (s *Source) MySQLPool() *sql.DB { return s.Pool } +func (s *Source) MySQLDatabase() string { + return s.Database +} + func (s *Source) RunSQL(ctx context.Context, statement string, params []any) (any, error) { results, err := s.MySQLPool().QueryContext(ctx, statement, params...) if err != nil { diff --git a/internal/sources/mysql/mysql.go b/internal/sources/mysql/mysql.go index 15d696f37a4d..477c9a982826 100644 --- a/internal/sources/mysql/mysql.go +++ b/internal/sources/mysql/mysql.go @@ -101,6 +101,10 @@ func (s *Source) MySQLPool() *sql.DB { return s.Pool } +func (s *Source) MySQLDatabase() string { + return s.Database +} + func (s *Source) RunSQL(ctx context.Context, statement string, params []any) (any, error) { results, err := s.MySQLPool().QueryContext(ctx, statement, params...) if err != nil { diff --git a/internal/tools/elasticsearch/elasticsearchesql/elasticsearchesql.go b/internal/tools/elasticsearch/elasticsearchesql/elasticsearchesql.go index 05a261981a2d..7e4992ddf97f 100644 --- a/internal/tools/elasticsearch/elasticsearchesql/elasticsearchesql.go +++ b/internal/tools/elasticsearch/elasticsearchesql/elasticsearchesql.go @@ -49,7 +49,7 @@ type Config struct { Source string `yaml:"source" validate:"required"` Description string `yaml:"description" validate:"required"` AuthRequired []string `yaml:"authRequired" validate:"required"` - Query string `yaml:"query"` + Query string `yaml:"query" validate:"required"` Format string `yaml:"format"` Timeout int `yaml:"timeout"` Parameters parameters.Parameters `yaml:"parameters"` @@ -109,25 +109,21 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } query := t.Query - sqlParams := make([]map[string]any, 0, len(params)) paramMap := params.AsMap() - // If a query is provided in the params and not already set in the tool, use it. - if queryVal, ok := paramMap["query"]; ok { - if str, ok := queryVal.(string); ok && t.Query == "" { - query = str - } - - // Drop the query param if not a string or if the tool already has a query. - delete(paramMap, "query") - } + var paramsList []map[string]any for _, param := range t.Parameters { if param.GetType() == "array" { return nil, util.NewAgentError("array parameters are not supported yet", nil) } - sqlParams = append(sqlParams, map[string]any{param.GetName(): paramMap[param.GetName()]}) + + // ES|QL requires an array of single-key objects for named parameters + if val, ok := paramMap[param.GetName()]; ok { + paramsList = append(paramsList, map[string]any{param.GetName(): val}) + } } - resp, err := source.RunSQL(ctx, t.Format, query, sqlParams) + + resp, err := source.RunSQL(ctx, t.Format, query, paramsList) if err != nil { return nil, util.ProcessGeneralError(err) } diff --git a/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats.go b/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats.go new file mode 100644 index 000000000000..c3a9f8a639b1 --- /dev/null +++ b/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats.go @@ -0,0 +1,217 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysqllisttablestats + +import ( + "context" + "database/sql" + "fmt" + "net/http" + + yaml "github.com/goccy/go-yaml" + "github.com/googleapis/mcp-toolbox/internal/embeddingmodels" + "github.com/googleapis/mcp-toolbox/internal/sources" + "github.com/googleapis/mcp-toolbox/internal/tools" + "github.com/googleapis/mcp-toolbox/internal/util" + "github.com/googleapis/mcp-toolbox/internal/util/parameters" +) + +const resourceType string = "mysql-list-table-stats" + +const listTableStatsStatement = ` +SELECT + t.table_schema AS 'table_schema', + t.table_name AS 'table_name', + ROUND((t.data_length + t.index_length) / 1024 / 1024, 2) AS 'size_MB', + t.TABLE_ROWS AS 'row_count', + ROUND(ts.total_latency / 1000000000000, 2) AS 'total_latency_secs', + ts.rows_fetched AS 'rows_fetched', + ts.rows_inserted AS 'rows_inserted', + ts.rows_updated AS 'rows_updated', + ts.rows_deleted AS 'rows_deleted', + ts.io_read_requests AS 'io_reads', + ROUND(ts.io_read_latency / 1000000000000, 2) AS 'io_read_latency', + ts.io_write_requests AS 'io_writes', + ROUND(ts.io_write_latency / 1000000000000, 2) AS 'io_write_latency', + ts.io_misc_requests AS 'io_misc_requests', + ROUND(ts.io_misc_latency / 1000000000000, 2) AS 'io_misc_latency' +FROM + information_schema.tables AS t +INNER JOIN + sys.x$schema_table_statistics AS ts + ON (t.table_schema = ts.table_schema AND t.table_name = ts.table_name) +WHERE + t.table_schema NOT IN ('sys', 'information_schema', 'mysql', 'performance_schema') + AND (t.table_schema = COALESCE(NULLIF(?, ''), NULLIF(DATABASE(), '')) OR COALESCE(NULLIF(?, ''), NULLIF(DATABASE(), '')) IS NULL) + AND (COALESCE(?, '') = '' OR t.table_name = ?) +ORDER BY + CASE ? + WHEN 'row_count' THEN row_count + WHEN 'rows_fetched' THEN rows_fetched + WHEN 'rows_inserted' THEN rows_inserted + WHEN 'rows_updated' THEN rows_updated + WHEN 'rows_deleted' THEN rows_deleted + ELSE ts.total_latency + END DESC +LIMIT ?; +` + +func init() { + if !tools.Register(resourceType, newConfig) { + panic(fmt.Sprintf("tool type %q already registered", resourceType)) + } +} + +func newConfig(ctx context.Context, name string, decoder *yaml.Decoder) (tools.ToolConfig, error) { + actual := Config{Name: name} + if err := decoder.DecodeContext(ctx, &actual); err != nil { + return nil, err + } + return actual, nil +} + +type compatibleSource interface { + MySQLPool() *sql.DB + RunSQL(context.Context, string, []any) (any, error) + MySQLDatabase() string +} + +type Config struct { + Name string `yaml:"name" validate:"required"` + Type string `yaml:"type" validate:"required"` + Source string `yaml:"source" validate:"required"` + Description string `yaml:"description" validate:"required"` + AuthRequired []string `yaml:"authRequired"` + Annotations *tools.ToolAnnotations `yaml:"annotations,omitempty"` +} + +// validate interface +var _ tools.ToolConfig = Config{} + +func (cfg Config) ToolConfigType() string { + return resourceType +} + +func (cfg Config) Initialize(srcs map[string]sources.Source) (tools.Tool, error) { + allParameters := parameters.Parameters{ + parameters.NewStringParameterWithDefault("table_schema", "", "(Optional) The database where statistics is to be executed. Check all tables visible to the current user if not specified"), + parameters.NewStringParameterWithDefault("table_name", "", "(Optional) Name of the table to be checked. Check all tables visible to the current user if not specified."), + parameters.NewStringParameterWithDefault("sort_by", "", "(Optional) The column to sort by"), + parameters.NewIntParameterWithDefault("limit", 10, "(Optional) Max rows to return, default is 10"), + parameters.NewStringParameterWithRequired("connected_schema", "(Optional) The connected db", false), + } + annotations := tools.GetAnnotationsOrDefault(cfg.Annotations, tools.NewReadOnlyAnnotations) + mcpManifest := tools.GetMcpManifest(cfg.Name, cfg.Description, cfg.AuthRequired, allParameters, annotations) + + // finish tool setup + t := Tool{ + Config: cfg, + allParams: allParameters, + manifest: tools.Manifest{Description: cfg.Description, Parameters: allParameters.Manifest(), AuthRequired: cfg.AuthRequired}, + mcpManifest: mcpManifest, + } + return t, nil +} + +// validate interface +var _ tools.Tool = Tool{} + +type Tool struct { + Config + allParams parameters.Parameters `yaml:"parameters"` + manifest tools.Manifest + mcpManifest tools.McpManifest +} + +func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, params parameters.ParamValues, accessToken tools.AccessToken) (any, util.ToolboxError) { + source, err := tools.GetCompatibleSource[compatibleSource](resourceMgr, t.Source, t.Name, t.Type) + if err != nil { + return nil, util.NewClientServerError("source used is not compatible with the tool", http.StatusInternalServerError, err) + } + + paramsMap := params.AsMap() + + table_schema, ok := paramsMap["table_schema"].(string) + if !ok { + return nil, util.NewAgentError("invalid 'table_schema' parameter; expected a string", nil) + } + table_name, ok := paramsMap["table_name"].(string) + if !ok { + return nil, util.NewAgentError("invalid 'table_name' parameter; expected a string", nil) + } + sort_by, ok := paramsMap["sort_by"].(string) + if !ok { + return nil, util.NewAgentError("invalid 'sort_by' parameter; expected a string", nil) + } + limit, ok := paramsMap["limit"].(int) + if !ok { + return nil, util.NewAgentError("invalid 'limit' parameter; expected an integer", nil) + } + // Validate connected schema is either skipped or same as queried schema + connected_schema, _ := paramsMap["connected_schema"].(string) + if connected_schema == "" { + connected_schema = source.MySQLDatabase() + } + if table_schema != connected_schema && connected_schema != "" && table_schema != "" { + err := fmt.Errorf("error: connected schema '%s' does not match queried schema '%s'", connected_schema, table_schema) + return nil, util.NewClientServerError("schema match failed", http.StatusInternalServerError, err) + } + + // Log the query executed for debugging. + logger, err := util.LoggerFromContext(ctx) + if err != nil { + return nil, util.NewClientServerError("error getting logger", http.StatusInternalServerError, err) + } + logger.DebugContext(ctx, fmt.Sprintf("executing `%s` tool query: %s", resourceType, listTableStatsStatement)) + sliceParams := []any{table_schema, table_schema, table_name, table_name, sort_by, limit} + resp, err := source.RunSQL(ctx, listTableStatsStatement, sliceParams) + if err != nil { + return nil, util.ProcessGeneralError(err) + } + + return resp, nil +} + +func (t Tool) EmbedParams(ctx context.Context, paramValues parameters.ParamValues, embeddingModelsMap map[string]embeddingmodels.EmbeddingModel) (parameters.ParamValues, error) { + return parameters.EmbedParams(ctx, t.allParams, paramValues, embeddingModelsMap, nil) +} + +func (t Tool) Manifest() tools.Manifest { + return t.manifest +} + +func (t Tool) McpManifest() tools.McpManifest { + return t.mcpManifest +} + +func (t Tool) Authorized(verifiedAuthServices []string) bool { + return tools.IsAuthorized(t.AuthRequired, verifiedAuthServices) +} + +func (t Tool) RequiresClientAuthorization(resourceMgr tools.SourceProvider) (bool, error) { + return false, nil +} + +func (t Tool) ToConfig() tools.ToolConfig { + return t.Config +} + +func (t Tool) GetAuthTokenHeaderName(resourceMgr tools.SourceProvider) (string, error) { + return "Authorization", nil +} + +func (t Tool) GetParameters() parameters.Parameters { + return t.allParams +} diff --git a/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats_test.go b/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats_test.go new file mode 100644 index 000000000000..bf5ccd3938d7 --- /dev/null +++ b/internal/tools/mysql/mysqllisttablestats/mysqllisttablestats_test.go @@ -0,0 +1,71 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysqllisttablestats_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/googleapis/mcp-toolbox/internal/server" + "github.com/googleapis/mcp-toolbox/internal/testutils" + "github.com/googleapis/mcp-toolbox/internal/tools/mysql/mysqllisttablestats" +) + +func TestParseFromYamlMySQLListTableStats(t *testing.T) { + ctx, err := testutils.ContextWithNewLogger() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + tcs := []struct { + desc string + in string + want server.ToolConfigs + }{ + { + desc: "basic example", + in: ` + kind: tool + name: example_tool + type: mysql-list-table-stats + source: my-mysql-instance + description: some description + authRequired: + - my-google-auth-service + - other-auth-service + `, + want: server.ToolConfigs{ + "example_tool": mysqllisttablestats.Config{ + Name: "example_tool", + Type: "mysql-list-table-stats", + Source: "my-mysql-instance", + Description: "some description", + AuthRequired: []string{"my-google-auth-service", "other-auth-service"}, + }, + }, + }, + } + for _, tc := range tcs { + t.Run(tc.desc, func(t *testing.T) { + // Parse contents + _, _, _, got, _, _, err := server.UnmarshalResourceConfig(ctx, testutils.FormatYaml(tc.in)) + if err != nil { + t.Fatalf("unable to unmarshal: %s", err) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Fatalf("incorrect parse: diff %v", diff) + } + }) + } +} diff --git a/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go b/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go index 64ae977e5d36..e65a351e14d9 100644 --- a/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go +++ b/internal/tools/singlestore/singlestoreexecutesql/singlestoreexecutesql.go @@ -128,7 +128,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } func (t Tool) EmbedParams(ctx context.Context, paramValues parameters.ParamValues, embeddingModelsMap map[string]embeddingmodels.EmbeddingModel) (parameters.ParamValues, error) { - return parameters.EmbedParams(ctx, t.Parameters, paramValues, embeddingModelsMap, nil) + return parameters.EmbedParams(ctx, t.Parameters, paramValues, embeddingModelsMap, embeddingmodels.FormatVectorForPgvector) } func (t Tool) Manifest() tools.Manifest { diff --git a/internal/tools/singlestore/singlestoresql/singlestoresql.go b/internal/tools/singlestore/singlestoresql/singlestoresql.go index e02118bc6df1..d65814b4dbbc 100644 --- a/internal/tools/singlestore/singlestoresql/singlestoresql.go +++ b/internal/tools/singlestore/singlestoresql/singlestoresql.go @@ -156,7 +156,7 @@ func (t Tool) Invoke(ctx context.Context, resourceMgr tools.SourceProvider, para } func (t Tool) EmbedParams(ctx context.Context, paramValues parameters.ParamValues, embeddingModelsMap map[string]embeddingmodels.EmbeddingModel) (parameters.ParamValues, error) { - return parameters.EmbedParams(ctx, t.AllParams, paramValues, embeddingModelsMap, nil) + return parameters.EmbedParams(ctx, t.AllParams, paramValues, embeddingModelsMap, embeddingmodels.FormatVectorForPgvector) } func (t Tool) Manifest() tools.Manifest { diff --git a/server.json b/server.json index 3f03ea457c10..3f961ca0e9d8 100644 --- a/server.json +++ b/server.json @@ -14,11 +14,11 @@ "url": "https://github.com/googleapis/mcp-toolbox", "source": "github" }, - "version": "0.32.0", + "version": "1.0.0", "packages": [ { "registryType": "oci", - "identifier": "us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:0.32.0", + "identifier": "us-central1-docker.pkg.dev/database-toolbox/toolbox/toolbox:1.0.0", "transport": { "type": "streamable-http", "url": "http://{host}:{port}/mcp" diff --git a/tests/cloudsqlmysql/cloud_sql_mysql_integration_test.go b/tests/cloudsqlmysql/cloud_sql_mysql_integration_test.go index f6660b8422c4..ef689f40cdb0 100644 --- a/tests/cloudsqlmysql/cloud_sql_mysql_integration_test.go +++ b/tests/cloudsqlmysql/cloud_sql_mysql_integration_test.go @@ -164,6 +164,7 @@ func TestCloudSQLMySQLToolEndpoints(t *testing.T) { tests.RunMySQLListTablesTest(t, CloudSQLMySQLDatabase, tableNameParam, tableNameAuth, expectedOwner) tests.RunMySQLListActiveQueriesTest(t, ctx, pool) tests.RunMySQLGetQueryPlanTest(t, ctx, pool, CloudSQLMySQLDatabase, tableNameParam) + tests.RunMySQLListTableStatsTest(t, ctx, pool, CloudSQLMySQLDatabase, tableNameParam, tableNameAuth) } // Test connection with different IP type diff --git a/tests/common.go b/tests/common.go index 870ef84b0f1e..db2e0deb812c 100644 --- a/tests/common.go +++ b/tests/common.go @@ -456,6 +456,11 @@ func AddMySQLPrebuiltToolConfig(t *testing.T, config map[string]any) map[string] "source": "my-instance", "description": "Lists table fragmentation in the database.", } + tools["list_table_stats"] = map[string]any{ + "type": "mysql-list-table-stats", + "source": "my-instance", + "description": "Lists table stats in the database.", + } tools["get_query_plan"] = map[string]any{ "type": "mysql-get-query-plan", "source": "my-instance", diff --git a/tests/elasticsearch/elasticsearch_integration_test.go b/tests/elasticsearch/elasticsearch_integration_test.go index bded6f319c12..b068d880e8d9 100644 --- a/tests/elasticsearch/elasticsearch_integration_test.go +++ b/tests/elasticsearch/elasticsearch_integration_test.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "net/http" - "os" "regexp" "strings" "testing" @@ -28,6 +27,8 @@ import ( "github.com/elastic/go-elasticsearch/v9" "github.com/elastic/go-elasticsearch/v9/esapi" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" "github.com/googleapis/mcp-toolbox/internal/testutils" "github.com/googleapis/mcp-toolbox/tests" @@ -36,15 +37,57 @@ import ( var ( ElasticsearchSourceType = "elasticsearch" ElasticsearchToolType = "elasticsearch-esql" - EsAddress = os.Getenv("ELASTICSEARCH_HOST") - EsUser = os.Getenv("ELASTICSEARCH_USER") - EsPass = os.Getenv("ELASTICSEARCH_PASS") + EsAddress = "" + EsUser = "elastic" + EsPass = "test-password" ) -func getElasticsearchVars(t *testing.T) map[string]any { - if EsAddress == "" { - t.Fatal("'ELASTICSEARCH_HOST' not set") +func setupElasticsearchContainer(ctx context.Context, t *testing.T) (string, func()) { + t.Helper() + + req := testcontainers.ContainerRequest{ + Image: "docker.elastic.co/elasticsearch/elasticsearch:9.3.2", + ExposedPorts: []string{"9200/tcp"}, + Env: map[string]string{ + "discovery.type": "single-node", + "xpack.security.enabled": "false", + }, + WaitingFor: wait.ForAll( + wait.ForHTTP("/"), + wait.ForExposedPort(), + ), } + + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + t.Fatalf("failed to start elasticsearch container: %s", err) + } + + cleanup := func() { + if err := container.Terminate(ctx); err != nil { + t.Fatalf("failed to terminate container: %s", err) + } + } + + host, err := container.Host(ctx) + if err != nil { + cleanup() + t.Fatalf("failed to get container host: %s", err) + } + + mappedPort, err := container.MappedPort(ctx, "9200") + if err != nil { + cleanup() + t.Fatalf("failed to get container mapped port: %s", err) + } + + return fmt.Sprintf("http://%s:%s", host, mappedPort.Port()), cleanup +} + +func getElasticsearchVars(t *testing.T) map[string]any { return map[string]any{ "type": ElasticsearchSourceType, "addresses": []string{EsAddress}, @@ -64,9 +107,13 @@ type ElasticsearchWants struct { } func TestElasticsearchToolEndpoints(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() + var containerCleanup func() + EsAddress, containerCleanup = setupElasticsearchContainer(ctx, t) + defer containerCleanup() + args := []string{"--enable-api"} sourceConfig := getElasticsearchVars(t) @@ -77,6 +124,10 @@ func TestElasticsearchToolEndpoints(t *testing.T) { toolsConfig := getElasticsearchToolsConfig(sourceConfig, ElasticsearchToolType, paramToolStatement, idParamToolStatement, nameParamToolStatement, arrayParamToolStatement, authToolStatement) + searchStmt := fmt.Sprintf(`FROM %s | WHERE KNN(embedding, ?query) | LIMIT 1 | KEEP id, name`, index) + insertStmt := fmt.Sprintf("FROM %s | WHERE name == ?content | EVAL dummy = ?text_to_embed | LIMIT 0", index) + toolsConfig = tests.AddSemanticSearchConfig(t, toolsConfig, ElasticsearchToolType, insertStmt, searchStmt) + cmd, cleanup, err := tests.StartCmd(ctx, toolsConfig, args...) if err != nil { t.Fatalf("failed to start cmd: %v", err) @@ -100,21 +151,57 @@ func TestElasticsearchToolEndpoints(t *testing.T) { t.Fatalf("error creating the Elasticsearch client: %s", err) } - // Delete index if already exists + // Delete indices if already exists defer func() { _, err = esapi.IndicesDeleteRequest{ Index: []string{index}, }.Do(ctx, esClient) if err != nil { - t.Fatalf("error deleting index: %s", err) + t.Errorf("error deleting indices: %s", err) } }() alice := fmt.Sprintf(`{ - "id": 1, - "name": "Alice", - "email": "%s" - }`, tests.ServiceAccountEmail) + "id": 1, + "name": "Alice", + "email": "%s" + }`, tests.ServiceAccountEmail) + + // Create index with mapping for vector search + mapping := `{ + "mappings": { + "properties": { + "embedding": { + "type": "dense_vector", + "dims": 768, + "index": true, + "similarity": "cosine" + } + } + } + }` + res, err := esapi.IndicesCreateRequest{ + Index: index, + Body: strings.NewReader(mapping), + }.Do(ctx, esClient) + if err != nil { + t.Fatalf("error creating index: %s", err) + } + if res.IsError() { + t.Logf("Create index response error (might be ignored): %s", res.String()) + } + + vectorSize := 768 + var sb strings.Builder + sb.WriteString("[") + for i := 0; i < vectorSize; i++ { + sb.WriteString("0.1") + if i < vectorSize-1 { + sb.WriteString(", ") + } + } + sb.WriteString("]") + semanticDoc := fmt.Sprintf(`{"id": 5, "name": "Semantic", "embedding": %s}`, sb.String()) // Index sample documents sampleDocs := []string{ @@ -122,6 +209,7 @@ func TestElasticsearchToolEndpoints(t *testing.T) { `{"id": 2, "name": "Jane", "email": "janedoe@gmail.com"}`, `{"id": 3, "name": "Sid"}`, `{"id": 4, "name": "null"}`, + semanticDoc, } for _, doc := range sampleDocs { res, err := esapi.IndexRequest{ @@ -150,27 +238,28 @@ func TestElasticsearchToolEndpoints(t *testing.T) { ) tests.RunMCPToolCallMethod(t, wants.McpMyFailTool, wants.McpSelect1, tests.WithMcpMyToolId3NameAliceWant(wants.McpMyToolId3NameAlice)) + // Semantic search tests + semanticSearchWant := `[{"id":5,"name":"Semantic"}]` + tests.RunSemanticSearchToolInvokeTest(t, "[]", "[]", semanticSearchWant) runExecuteEsqlTest(t, index) - } func getElasticsearchQueries(index string) (string, string, string, string, string) { - paramToolStatement := fmt.Sprintf(`FROM %s | WHERE id == ?id OR name == ?name | SORT id ASC`, index) - idParamToolStatement := fmt.Sprintf(`FROM %s | WHERE id == ?id`, index) - nameParamToolStatement := fmt.Sprintf(`FROM %s | WHERE name == ?name`, index) - arrayParamToolStatement := fmt.Sprintf(`FROM %s | WHERE first_name == ?first_name_array`, index) // Not supported yet. + paramToolStatement := fmt.Sprintf(`FROM %s | WHERE id == ?id OR name == ?name | SORT id ASC | KEEP id, name, name.keyword, email, email.keyword`, index) + idParamToolStatement := fmt.Sprintf(`FROM %s | WHERE id == ?id | KEEP id, name, name.keyword, email, email.keyword`, index) + nameParamToolStatement := fmt.Sprintf(`FROM %s | WHERE name == ?name | KEEP id, name, name.keyword, email, email.keyword`, index) authToolStatement := fmt.Sprintf(`FROM %s | WHERE email == ?email | KEEP name`, index) - return paramToolStatement, idParamToolStatement, nameParamToolStatement, arrayParamToolStatement, authToolStatement + return paramToolStatement, idParamToolStatement, nameParamToolStatement, "", authToolStatement } func getElasticsearchWants() ElasticsearchWants { - select1Want := fmt.Sprintf(`[{"email":"%[1]s","email.keyword":"%[1]s","id":1,"name":"Alice","name.keyword":"Alice"},{"email":"janedoe@gmail.com","email.keyword":"janedoe@gmail.com","id":2,"name":"Jane","name.keyword":"Jane"},{"email":null,"email.keyword":null,"id":3,"name":"Sid","name.keyword":"Sid"},{"email":null,"email.keyword":null,"id":4,"name":"null","name.keyword":"null"}]`, tests.ServiceAccountEmail) + select1Want := fmt.Sprintf(`[{"email":"%[1]s","email.keyword":"%[1]s","id":1,"name":"Alice","name.keyword":"Alice"},{"email":"janedoe@gmail.com","email.keyword":"janedoe@gmail.com","id":2,"name":"Jane","name.keyword":"Jane"},{"email":null,"email.keyword":null,"id":3,"name":"Sid","name.keyword":"Sid"},{"email":null,"email.keyword":null,"id":4,"name":"null","name.keyword":"null"},{"email":null,"email.keyword":null,"id":5,"name":"Semantic","name.keyword":"Semantic"}]`, tests.ServiceAccountEmail) myToolId3NameAliceWant := fmt.Sprintf(`[{"email":"%[1]s","email.keyword":"%[1]s","id":1,"name":"Alice","name.keyword":"Alice"},{"email":null,"email.keyword":null,"id":3,"name":"Sid","name.keyword":"Sid"}]`, tests.ServiceAccountEmail) myToolById4Want := `[{"email":null,"email.keyword":null,"id":4,"name":"null","name.keyword":"null"}]` nullWant := `{"error":{"root_cause":[{"type":"verification_exception","reason":"Found 1 problem\nline 1:25: first argument of [name == ?name] is [text] so second argument must also be [text] but was [null]"}],"type":"verification_exception","reason":"Found 1 problem\nline 1:25: first argument of [name == ?name] is [text] so second argument must also be [text] but was [null]"},"status":400}` - mcpMyFailToolWant := `{"content":[{"type":"text","text":"{\"error\":{\"root_cause\":[{\"type\":\"parsing_exception\",\"reason\":\"line 1:1: mismatched input 'SELEC' expecting {, 'row', 'from', 'show'}\"}],\"type\":\"parsing_exception\",\"reason\":\"line 1:1: mismatched input 'SELEC' expecting {, 'row', 'from', 'show'}\",\"caused_by\":{\"type\":\"input_mismatch_exception\",\"reason\":null}},\"status\":400}"}]}` + mcpMyFailToolWant := `{"content":[{"type":"text","text":"{\"error\":{\"root_cause\":[{\"type\":\"parsing_exception\",\"reason\":\"line 1:1: mismatched input 'SELEC' expecting {, 'row', 'from', 'ts', 'set', 'show'}\"}],\"type\":\"parsing_exception\",\"reason\":\"line 1:1: mismatched input 'SELEC' expecting {, 'row', 'from', 'ts', 'set', 'show'}\",\"caused_by\":{\"type\":\"input_mismatch_exception\",\"reason\":null}},\"status\":400}"}]}` mcpMyToolId3NameAliceWant := fmt.Sprintf(`{"jsonrpc":"2.0","id":"my-tool","result":{"content":[{"type":"text","text":"[{\"email\":\"%[1]s\",\"email.keyword\":\"%[1]s\",\"id\":1,\"name\":\"Alice\",\"name.keyword\":\"Alice\"},{\"email\":null,\"email.keyword\":null,\"id\":3,\"name\":\"Sid\",\"name.keyword\":\"Sid\"}]"}]}}`, tests.ServiceAccountEmail) - mcpSelect1Want := fmt.Sprintf(`{"jsonrpc":"2.0","id":"invoke my-auth-required-tool","result":{"content":[{"type":"text","text":"[{\"email\":\"%[1]s\",\"email.keyword\":\"%[1]s\",\"id\":1,\"name\":\"Alice\",\"name.keyword\":\"Alice\"},{\"email\":\"janedoe@gmail.com\",\"email.keyword\":\"janedoe@gmail.com\",\"id\":2,\"name\":\"Jane\",\"name.keyword\":\"Jane\"},{\"email\":null,\"email.keyword\":null,\"id\":3,\"name\":\"Sid\",\"name.keyword\":\"Sid\"},{\"email\":null,\"email.keyword\":null,\"id\":4,\"name\":\"null\",\"name.keyword\":\"null\"}]"}]}}`, tests.ServiceAccountEmail) + mcpSelect1Want := fmt.Sprintf(`{"jsonrpc":"2.0","id":"invoke my-auth-required-tool","result":{"content":[{"type":"text","text":"[{\"email\":\"%[1]s\",\"email.keyword\":\"%[1]s\",\"id\":1,\"name\":\"Alice\",\"name.keyword\":\"Alice\"},{\"email\":\"janedoe@gmail.com\",\"email.keyword\":\"janedoe@gmail.com\",\"id\":2,\"name\":\"Jane\",\"name.keyword\":\"Jane\"},{\"email\":null,\"email.keyword\":null,\"id\":3,\"name\":\"Sid\",\"name.keyword\":\"Sid\"},{\"email\":null,\"email.keyword\":null,\"id\":4,\"name\":\"null\",\"name.keyword\":\"null\"},{\"email\":null,\"email.keyword\":null,\"id\":5,\"name\":\"Semantic\",\"name.keyword\":\"Semantic\"}]"}]}}`, tests.ServiceAccountEmail) return ElasticsearchWants{ Select1: select1Want, @@ -199,7 +288,7 @@ func getElasticsearchToolsConfig(sourceConfig map[string]any, toolType, paramToo "type": toolType, "source": "my-instance", "description": "Simple tool to test end to end functionality.", - "query": "FROM test-index | SORT id ASC", + "query": "FROM test-index | SORT id ASC | KEEP id, name, name.keyword, email, email.keyword", }, "my-tool": map[string]any{ "type": toolType, @@ -246,34 +335,6 @@ func getElasticsearchToolsConfig(sourceConfig map[string]any, toolType, paramToo }, }, }, - "my-array-tool": map[string]any{ - "type": toolType, - "source": "my-instance", - "description": "Tool to test invocation with array params.", - "query": arrayToolStatement, - "parameters": []any{ - map[string]any{ - "name": "idArray", - "type": "array", - "description": "ID array", - "items": map[string]any{ - "name": "id", - "type": "integer", - "description": "ID", - }, - }, - map[string]any{ - "name": "nameArray", - "type": "array", - "description": "user name array", - "items": map[string]any{ - "name": "name", - "type": "string", - "description": "user name", - }, - }, - }, - }, "my-auth-tool": map[string]any{ "type": toolType, "source": "my-instance", @@ -298,7 +359,7 @@ func getElasticsearchToolsConfig(sourceConfig map[string]any, toolType, paramToo "type": toolType, "source": "my-instance", "description": "Tool to test auth required invocation.", - "query": "FROM test-index | SORT id ASC", + "query": "FROM test-index | SORT id ASC | KEEP id, name, name.keyword, email, email.keyword", "authRequired": []string{ "my-google-auth", }, @@ -339,7 +400,7 @@ func runExecuteEsqlTest(t *testing.T, index string) { if !ok { t.Fatalf("unable to find result in response body") } - want := `[{"id":1},{"id":2},{"id":3},{"id":4}]` + want := `[{"id":1},{"id":2},{"id":3},{"id":4},{"id":5}]` if got != want { t.Fatalf("unexpected value: got %q, want %q", got, want) } diff --git a/tests/embedding.go b/tests/embedding.go index 0e54d5db4f32..adc10640016e 100644 --- a/tests/embedding.go +++ b/tests/embedding.go @@ -52,11 +52,16 @@ func AddSemanticSearchConfig(t *testing.T, config map[string]any, toolKind, inse t.Fatalf("unable to get tools from config") } + queryKey := "statement" + if toolKind == "elasticsearch-esql" { + queryKey = "query" + } + tools["insert_docs"] = map[string]any{ "kind": toolKind, "source": "my-instance", "description": "Stores content and its vector embedding into the documents table.", - "statement": insertStmt, + queryKey: insertStmt, "parameters": []any{ map[string]any{ "name": "content", @@ -77,7 +82,7 @@ func AddSemanticSearchConfig(t *testing.T, config map[string]any, toolKind, inse "kind": toolKind, "source": "my-instance", "description": "Finds the most semantically similar document to the query vector.", - "statement": searchStmt, + queryKey: searchStmt, "parameters": []any{ map[string]any{ "name": "query", diff --git a/tests/mysql/mysql_integration_test.go b/tests/mysql/mysql_integration_test.go index 1e2ce35f8363..de30c796e24c 100644 --- a/tests/mysql/mysql_integration_test.go +++ b/tests/mysql/mysql_integration_test.go @@ -144,4 +144,5 @@ func TestMySQLToolEndpoints(t *testing.T) { tests.RunMySQLListTablesMissingUniqueIndexes(t, ctx, pool, MySQLDatabase) tests.RunMySQLListTableFragmentationTest(t, MySQLDatabase, tableNameParam, tableNameAuth) tests.RunMySQLGetQueryPlanTest(t, ctx, pool, MySQLDatabase, tableNameParam) + tests.RunMySQLListTableStatsTest(t, ctx, pool, MySQLDatabase, tableNameParam, tableNameAuth) } diff --git a/tests/neo4j/neo4j_integration_test.go b/tests/neo4j/neo4j_integration_test.go index d694198294fb..8d61afd5b80c 100644 --- a/tests/neo4j/neo4j_integration_test.go +++ b/tests/neo4j/neo4j_integration_test.go @@ -77,44 +77,56 @@ func TestNeo4jToolEndpoints(t *testing.T) { // This configuration defines the data source and the tools to be tested. toolsFile := map[string]any{ "sources": map[string]any{ - "my-neo4j-instance": sourceConfig, + "my-instance": sourceConfig, }, "tools": map[string]any{ "my-simple-cypher-tool": map[string]any{ "type": "neo4j-cypher", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "Simple tool to test end to end functionality.", "statement": "RETURN 1 as a;", }, "my-simple-execute-cypher-tool": map[string]any{ "type": "neo4j-execute-cypher", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "Simple tool to test end to end functionality.", }, "my-readonly-execute-cypher-tool": map[string]any{ "type": "neo4j-execute-cypher", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "A readonly cypher execution tool.", "readOnly": true, }, "my-schema-tool": map[string]any{ "type": "neo4j-schema", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "A tool to get the Neo4j schema.", }, "my-schema-tool-with-cache": map[string]any{ "type": "neo4j-schema", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "A schema tool with a custom cache expiration.", "cacheExpireMinutes": 10, }, "my-populated-schema-tool": map[string]any{ "type": "neo4j-schema", - "source": "my-neo4j-instance", + "source": "my-instance", "description": "A tool to get the Neo4j schema from a populated DB.", }, }, } + + insertStmt := `CREATE (n:SenseAIDocument {content: $content, embedding: $text_to_embed}) RETURN 1 as result` + searchStmt := ` + MATCH (n:SenseAIDocument) + WITH n, vector.similarity.cosine(n.embedding, $query) AS score + WHERE score IS NOT NULL + ORDER BY score DESC + LIMIT 1 + RETURN n.content as content + ` + toolsFile = tests.AddSemanticSearchConfig(t, toolsFile, "neo4j-cypher", insertStmt, searchStmt) + cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...) if err != nil { t.Fatalf("command initialization returned an error: %s", err) @@ -578,4 +590,9 @@ func TestNeo4jToolEndpoints(t *testing.T) { } }) } + + // Semantic search tests + semanticInsertWant := `[{"result":1}]` + semanticSearchWant := `[{"content":"The quick brown fox jumps over the lazy dog"}]` + tests.RunSemanticSearchToolInvokeTest(t, semanticInsertWant, semanticInsertWant, semanticSearchWant) } diff --git a/tests/singlestore/singlestore_integration_test.go b/tests/singlestore/singlestore_integration_test.go index f88c89c15ea8..d0aecfdf0d85 100644 --- a/tests/singlestore/singlestore_integration_test.go +++ b/tests/singlestore/singlestore_integration_test.go @@ -263,6 +263,9 @@ func TestSingleStoreToolEndpoints(t *testing.T) { tmplSelectCombined, tmplSelectFilterCombined := getSingleStoreTmplToolStatement() toolsFile = tests.AddTemplateParamConfig(t, toolsFile, SingleStoreToolType, tmplSelectCombined, tmplSelectFilterCombined, "") + insertStmt := `INSERT INTO senseai_docs (content, embedding) VALUES (?, JSON_ARRAY_PACK(?))` + searchStmt := `SELECT content FROM senseai_docs ORDER BY DOT_PRODUCT(embedding, JSON_ARRAY_PACK(?)) DESC LIMIT 1` + toolsFile = tests.AddSemanticSearchConfig(t, toolsFile, SingleStoreToolType, insertStmt, searchStmt) cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...) if err != nil { t.Fatalf("command initialization returned an error: %s", err) @@ -286,4 +289,22 @@ func TestSingleStoreToolEndpoints(t *testing.T) { tests.RunMCPToolCallMethod(t, mcpMyFailToolWant, mcpSelect1Want) tests.RunExecuteSqlToolInvokeTest(t, createTableStatement, select1Want) tests.RunToolInvokeWithTemplateParameters(t, tableNameTemplateParam) + + // Create table for semantic search + _, err = pool.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS senseai_docs (id INT AUTO_INCREMENT PRIMARY KEY, content TEXT, embedding BLOB);") + if err != nil { + t.Fatalf("unable to create semantic search table: %s", err) + } + defer func() { + _, err = pool.ExecContext(ctx, "DROP TABLE IF EXISTS senseai_docs;") + if err != nil { + t.Logf("Teardown failed: %s", err) + } + }() + + // Semantic search tests + httpSemanticInsertWant := `null` + mcpSemanticInsertWant := `` + semanticSearchWant := `The quick brown fox jumps over the lazy dog` + tests.RunSemanticSearchToolInvokeTest(t, httpSemanticInsertWant, mcpSemanticInsertWant, semanticSearchWant) } diff --git a/tests/spanner/spanner_integration_test.go b/tests/spanner/spanner_integration_test.go index 95ad32b42f12..4da0e73150f8 100644 --- a/tests/spanner/spanner_integration_test.go +++ b/tests/spanner/spanner_integration_test.go @@ -42,6 +42,7 @@ var ( SpannerProject = os.Getenv("SPANNER_PROJECT") SpannerDatabase = os.Getenv("SPANNER_DATABASE") SpannerInstance = os.Getenv("SPANNER_INSTANCE") + SpannerPgDatabase = os.Getenv("SPANNER_PG_DATABASE") ) func getSpannerVars(t *testing.T) map[string]any { @@ -162,6 +163,14 @@ func TestSpannerToolEndpoints(t *testing.T) { toolsFile = addSpannerListTablesConfig(t, toolsFile) toolsFile = addSpannerListGraphsConfig(t, toolsFile) + // Set up table for semantic search + vectorTableName, tearDownVectorTable := setupSpannerVectorTable(t, ctx, adminClient, dbString) + defer tearDownVectorTable(t) + + // Add semantic search tool config + insertStmt, searchStmt := getSpannerVectorSearchStmts(vectorTableName) + toolsFile = tests.AddSemanticSearchConfig(t, toolsFile, SpannerToolType, insertStmt, searchStmt) + cmd, cleanup, err := tests.StartCmd(ctx, toolsFile, args...) if err != nil { t.Fatalf("command initialization returned an error: %s", err) @@ -205,6 +214,7 @@ func TestSpannerToolEndpoints(t *testing.T) { runSpannerExecuteSqlToolInvokeTest(t, select1Want, invokeParamWant, tableNameParam) runSpannerListTablesTest(t, tableNameParam, tableNameAuth, tableNameTemplateParam) runSpannerListGraphsTest(t, graphName) + tests.RunSemanticSearchToolInvokeTest(t, "null", "", "The quick brown fox") } // getSpannerToolInfo returns statements and param for my-tool for spanner-sql type @@ -840,6 +850,50 @@ func runSpannerListGraphsTest(t *testing.T, graphName string) { } } +// setupSpannerVectorTable creates a vector table in Spanner for semantic search testing +func setupSpannerVectorTable(t *testing.T, ctx context.Context, adminClient *database.DatabaseAdminClient, dbString string) (string, func(*testing.T)) { + tableName := "vector_table_" + strings.ReplaceAll(uuid.New().String(), "-", "") + createStatement := fmt.Sprintf(`CREATE TABLE %s ( + id INT64, + content STRING(MAX), + embedding ARRAY + ) PRIMARY KEY (id)`, tableName) + + op, err := adminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ + Database: dbString, + Statements: []string{createStatement}, + }) + if err != nil { + t.Fatalf("unable to start create vector table operation %s: %s", tableName, err) + } + err = op.Wait(ctx) + if err != nil { + t.Fatalf("unable to create test vector table %s: %s", tableName, err) + } + + return tableName, func(t *testing.T) { + op, err = adminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ + Database: dbString, + Statements: []string{fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)}, + }) + if err != nil { + t.Errorf("unable to start drop %s operation: %s", tableName, err) + return + } + opErr := op.Wait(ctx) + if opErr != nil { + t.Errorf("Teardown failed: %s", opErr) + } + } +} + +// getSpannerVectorSearchStmts returns statements for spanner semantic search +func getSpannerVectorSearchStmts(vectorTableName string) (string, string) { + insertStmt := fmt.Sprintf("INSERT INTO %s (id, content, embedding) VALUES (1, @content, @text_to_embed)", vectorTableName) + searchStmt := fmt.Sprintf("SELECT id, content, COSINE_DISTANCE(embedding, @query) AS distance FROM %s ORDER BY distance LIMIT 1", vectorTableName) + return insertStmt, searchStmt +} + func runSpannerSchemaToolInvokeTest(t *testing.T, accessSchemaWant string) { invokeTcs := []struct { name string @@ -908,3 +962,116 @@ func runSpannerSchemaToolInvokeTest(t *testing.T, accessSchemaWant string) { }) } } + +// setupSpannerPgVectorTable creates a vector table in Spanner (PostgreSQL dialect) for semantic search testing +func setupSpannerPgVectorTable(t *testing.T, ctx context.Context, adminClient *database.DatabaseAdminClient, dbString string) (string, func(*testing.T)) { + tableName := "vector_table_" + strings.ReplaceAll(uuid.New().String(), "-", "") + createStatement := fmt.Sprintf(`CREATE TABLE %s ( + id bigint PRIMARY KEY, + content text, + embedding float4[] + )`, tableName) + + op, err := adminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ + Database: dbString, + Statements: []string{createStatement}, + }) + if err != nil { + t.Fatalf("unable to start create vector table operation %s: %s", tableName, err) + } + err = op.Wait(ctx) + if err != nil { + t.Fatalf("unable to create test vector table %s: %s", tableName, err) + } + + return tableName, func(t *testing.T) { + op, err = adminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ + Database: dbString, + Statements: []string{fmt.Sprintf("DROP TABLE IF EXISTS %s", tableName)}, + }) + if err != nil { + t.Errorf("unable to start drop %s operation: %s", tableName, err) + return + } + opErr := op.Wait(ctx) + if opErr != nil { + t.Errorf("Teardown failed: %s", opErr) + } + } +} + +// getSpannerPgVectorSearchStmts returns statements for spanner semantic search (PostgreSQL dialect) +func getSpannerPgVectorSearchStmts(vectorTableName string) (string, string) { + insertStmt := fmt.Sprintf("INSERT INTO %s (id, content, embedding) VALUES (1, $1, $2)", vectorTableName) + searchStmt := fmt.Sprintf("SELECT id, content, spanner.cosine_distance(embedding, $1::float4[]) AS distance FROM %s ORDER BY distance LIMIT 1", vectorTableName) + return insertStmt, searchStmt +} + +func TestSpannerPostgresqlToolEndpoints(t *testing.T) { + // Skip if environment variables are not set + if SpannerProject == "" || SpannerInstance == "" { + t.Skip("SPANNER_PROJECT or SPANNER_INSTANCE not set, skipping PostgreSQL tests") + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + + // Create admin client + adminClient, err := database.NewDatabaseAdminClient(ctx) + if err != nil { + t.Fatalf("unable to create admin client: %s", err) + } + defer adminClient.Close() + + dbName := SpannerPgDatabase + if dbName == "" { + dbName = "pg_test_database" + } + t.Logf("Using PostgreSQL database: %s", dbName) + + dbString := fmt.Sprintf("projects/%s/instances/%s/databases/%s", SpannerProject, SpannerInstance, dbName) + dataClient, err := spanner.NewClient(ctx, dbString) + if err != nil { + t.Fatalf("unable to create data client: %s", err) + } + defer dataClient.Close() + + // Set up table for semantic search + vectorTableName, tearDownVectorTable := setupSpannerPgVectorTable(t, ctx, adminClient, dbString) + defer tearDownVectorTable(t) + + // Add semantic search tool config + insertStmt, searchStmt := getSpannerPgVectorSearchStmts(vectorTableName) + + config := map[string]any{ + "sources": map[string]any{ + "my-instance": map[string]any{ + "type": "spanner", + "project": SpannerProject, + "instance": SpannerInstance, + "database": dbName, + "dialect": "postgresql", + }, + }, + "tools": map[string]any{}, + } + + toolsConfig := tests.AddSemanticSearchConfig(t, config, SpannerToolType, insertStmt, searchStmt) + + cmd, cleanup, err := tests.StartCmd(ctx, toolsConfig) + if err != nil { + t.Fatalf("command initialization returned an error: %s", err) + } + defer cleanup() + + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := testutils.WaitForString(waitCtx, regexp.MustCompile(`Server ready to serve`), cmd.Out) + if err != nil { + t.Logf("toolbox command logs: \n%s", out) + t.Fatalf("toolbox didn't start successfully: %s", err) + } + + // Run semantic search test + tests.RunSemanticSearchToolInvokeTest(t, "null", "", "The quick brown fox") +} diff --git a/tests/tool.go b/tests/tool.go index 3fc087c3e83b..5858370aa36f 100644 --- a/tests/tool.go +++ b/tests/tool.go @@ -3280,6 +3280,140 @@ func RunMySQLListTablesMissingUniqueIndexes(t *testing.T, ctx context.Context, p } } +func RunMySQLListTableStatsTest(t *testing.T, ctx context.Context, pool *sql.DB, databaseName string, tableNameParam string, tableNameAuth string) { + type tableStatsDetails struct { + TableSchema string `json:"table_schema"` + TableName string `json:"table_name"` + DataSize any `json:"size_MB"` + RowCount any `json:"row_count"` + TotalLatency any `json:"total_latency_secs"` + RowsFetched any `json:"rows_fetched"` + RowsInserted any `json:"rows_inserted"` + RowsUpdated any `json:"rows_updated"` + RowsDeleted any `json:"rows_deleted"` + IOReads any `json:"io_reads"` + IOReadLatency any `json:"io_read_latency"` + IOWriteLatency any `json:"io_write_latency"` + IOMiscLatency any `json:"io_misc_latency"` + } + + paramTableEntryWanted := tableStatsDetails{ + TableSchema: databaseName, + TableName: tableNameParam, + DataSize: any(nil), + RowCount: any(nil), + TotalLatency: any(nil), + RowsFetched: any(nil), + RowsInserted: any(nil), + RowsUpdated: any(nil), + RowsDeleted: any(nil), + IOReads: any(nil), + IOReadLatency: any(nil), + IOWriteLatency: any(nil), + IOMiscLatency: any(nil), + } + + authTableEntryWanted := tableStatsDetails{ + TableSchema: databaseName, + TableName: tableNameAuth, + DataSize: any(nil), + RowCount: any(nil), + TotalLatency: any(nil), + RowsFetched: any(nil), + RowsInserted: any(nil), + RowsUpdated: any(nil), + RowsDeleted: any(nil), + IOReads: any(nil), + IOReadLatency: any(nil), + IOWriteLatency: any(nil), + IOMiscLatency: any(nil), + } + + invokeTcs := []struct { + name string + requestBody io.Reader + wantStatusCode int + want any + }{ + { + name: "invoke list_table_stats with no arguments, expected 2 results", + requestBody: bytes.NewBufferString(`{}`), + wantStatusCode: http.StatusOK, + want: []tableStatsDetails{paramTableEntryWanted, authTableEntryWanted}, + }, + { + name: "invoke list_table_stats with schema other than connected to, expected log error and nil results", + requestBody: bytes.NewBufferString(fmt.Sprintf(`{"table_schema": "%s"}`, "somerandomdb_xyx")), + wantStatusCode: http.StatusInternalServerError, + want: []tableStatsDetails(nil), + }, + { + name: "invoke list_table_stats on 1 database and all tables, expected to have 2 result", + requestBody: bytes.NewBufferString(fmt.Sprintf(`{"table_schema": "%s"}`, databaseName)), + wantStatusCode: http.StatusOK, + want: []tableStatsDetails{paramTableEntryWanted, authTableEntryWanted}, + }, + { + name: "invoke list_table_stats on 1 database and 1 specific table name, expected to have 1 result", + requestBody: bytes.NewBufferString(fmt.Sprintf(`{"table_schema": "%s", "table_name": "%s"}`, databaseName, tableNameAuth)), + wantStatusCode: http.StatusOK, + want: []tableStatsDetails{authTableEntryWanted}, + }, + { + name: "invoke list_table_stats on 1 non-exist table on 1 database, expected to have 0 result", + requestBody: bytes.NewBufferString(`{"table_name": "non_existent_table"}`), + wantStatusCode: http.StatusOK, + want: []tableStatsDetails(nil), + }, + } + + // Generating additional stats for tableNameParam + for i := 0; i < 3; i++ { + selectStmt := fmt.Sprintf("SELECT * FROM %s", tableNameParam) + if _, err := pool.ExecContext(ctx, selectStmt); err != nil { + t.Logf("warning: unable to execute select: %v", err) + } + } + + for _, tc := range invokeTcs { + t.Run(tc.name, func(t *testing.T) { + const api = "http://127.0.0.1:5000/api/tool/list_table_stats/invoke" + resp, respBody := RunRequest(t, http.MethodPost, api, tc.requestBody, nil) + if resp.StatusCode != tc.wantStatusCode { + t.Fatalf("wrong status code: got %d, want %d, body: %s", resp.StatusCode, tc.wantStatusCode, string(respBody)) + } + if tc.wantStatusCode != http.StatusOK { + return + } + + var bodyWrapper struct { + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(respBody, &bodyWrapper); err != nil { + t.Fatalf("error decoding response wrapper: %v", err) + } + + var resultString string + if err := json.Unmarshal(bodyWrapper.Result, &resultString); err != nil { + resultString = string(bodyWrapper.Result) + } + + var got any + var details []tableStatsDetails + if err := json.Unmarshal([]byte(resultString), &details); err != nil { + t.Fatalf("failed to unmarshal outer JSON array into []tableInfo: %v", err) + } + got = details + + if diff := cmp.Diff(tc.want, got, cmp.Comparer(func(a, b tableStatsDetails) bool { + return a.TableSchema == b.TableSchema && a.TableName == b.TableName + })); diff != "" { + t.Errorf("Unexpected result: got %#v, want: %#v", got, tc.want) + } + }) + } +} + func RunMySQLListTableFragmentationTest(t *testing.T, databaseName, tableNameParam, tableNameAuth string) { type tableFragmentationDetails struct { TableSchema string `json:"table_schema"`