Merge branch 'trunk' into jeadie/openapi-10272

Author: lukekim

website/docs/api/HTTP/mcp-event.api.mdx

@@ -1,7 +1,7 @@
 ---
 id: mcp-event
 title: "Send message to MCP server"
-description: "Send message to the MCP endoint, for a given session."
+description: "Send a message to the MCP endpoint for a given session. Include the Mcp-Session-Id header to identify the session."
 sidebar_label: "Send message to MCP server"
 hide_title: true
 hide_table_of_contents: true
@@ -29,15 +29,15 @@ import Heading from "@theme/Heading";
 
 <MethodEndpoint
   method={"post"}
-  path={"/v1/mcp/sse"}
+  path={"/v1/mcp"}
   context={"endpoint"}
 >
 
 </MethodEndpoint>
 
 
 
-Send message to the MCP endoint, for a given session.
+Send a message to the MCP endpoint for a given session. Include the `Mcp-Session-Id` header to identify the session.
 
 <Heading
   id={"request"}
@@ -48,7 +48,7 @@ Send message to the MCP endoint, for a given session.
 </Heading>
 
 <ParamsDetails
-  parameters={[{"name":"sessionId","in":"query","required":true}]}
+  parameters={[{"name":"Mcp-Session-Id","in":"header","required":true}]}
 >
 
 </ParamsDetails>
@@ -63,7 +63,7 @@ Send message to the MCP endoint, for a given session.
 <StatusCodes
   id={undefined}
   label={undefined}
-  responses={{"202":{"description":"Message accepted. Response will stream via SSE."},"404":{"description":"Session not found. No active session for the given `session_id`."},"413":{"description":"Payload too large. Maximum allowed size is 4MB."},"500":{"description":"Internal server error. An unexpected issue occurred."}}}
+  responses={{"202":{"description":"Message accepted. Response will stream via the Streamable HTTP connection."},"404":{"description":"Session not found. No active session for the given `Mcp-Session-Id`."},"413":{"description":"Payload too large. Maximum allowed size is 32 MiB."},"500":{"description":"Internal server error. An unexpected issue occurred."}}}
 >
 
 </StatusCodes>

website/docs/api/HTTP/operation-id.api.mdx

@@ -1,8 +1,8 @@
 ---
 id: operation-id
-title: "Establish an MCP SSE Connection"
-description: "Initiates a Server-Sent Events (SSE) connection using the Model Context Protocol (MCP) to interact with Spice tools."
-sidebar_label: "Establish an MCP SSE Connection"
+title: "Establish an MCP Streamable HTTP Connection"
+description: "Initiates a Streamable HTTP connection using the Model Context Protocol (MCP) to interact with Spice tools."
+sidebar_label: "Establish an MCP Streamable HTTP Connection"
 hide_title: true
 hide_table_of_contents: true
 api: eJzlU8Fu2zAM/RWCpxZwnGanzbchCIYCKxrA2akLGlVmYmKypEm0t8Lwvw90tjbrPmHWxdYT36Men0cUc8pYPWBnI+4LbCjbxFE4eKzw1rOwEcpgoKY0UFrU5AU2A3nJcFXXm2uwwXuyWgF9Zn8CaQnuQkMO1sEL/RTYpiDBBgdXd+vtNUgA9kLJWIEfLC3UkS2BhOBy+dXrgsvn3lv6I0NNAdbxrG+Nh0y+gY5yNifKMLCBw/a+3sFyWC07G5c50wGMbyCRJR4IEuUYfKYM0qbQn1qQljPU9QayJDJdiQWGSMnolW4brF6/HrnBAl8YsBqnAjuSNuixEwkWGI20WOGFPhaYZ/PU6BH75LDCViRWy6UL1rg2ZKne33y4wbcD+KwwNDSQC7FT689MpVp75FOf6OzgYbFQxkOJ017lbJ9Ynmc9E/nxG+n7XjH2x4DViMLiCCtMvRfu6B/pXUuQdS4NvB5xbMln0npvOi3/GI1tafGu1Oa5IS98ZEp/I1OBev0z8apcrcqbRe+zmCdHCsaQpTP+gnYzg5xbMB7u1tt5PuuXpL3tdkSrUfPyP4dWnqNap80vozPs1do5bePvVD7gsMJi/tcL1GTuC9TwKTKOTybTl+SmSbe/95TmzBQ4mMTzpDRBF3n/tNnhNP0CfeNyCQ==
@@ -23,24 +23,24 @@ import Heading from "@theme/Heading";
 <Heading
   as={"h1"}
   className={"openapi__heading"}
-  children={"Establish an MCP SSE Connection"}
+  children={"Establish an MCP Streamable HTTP Connection"}
 >
 </Heading>
 
 <MethodEndpoint
   method={"get"}
-  path={"/v1/mcp/sse"}
+  path={"/v1/mcp"}
   context={"endpoint"}
 >
 
 </MethodEndpoint>
 
 
 
-Initiates a Server-Sent Events (SSE) connection using the Model Context Protocol (MCP) to interact with Spice tools.
+Initiates a Streamable HTTP connection using the Model Context Protocol (MCP) to interact with Spice tools.
 
 
-             Once connected, clients can send messages via `POST /v1/mcp/sse` and receive responses through this SSE stream.
+             Once connected, clients can send messages via `POST /v1/mcp` with an `Mcp-Session-Id` header and receive responses through this stream.
 
 <ParamsDetails
   parameters={undefined}

website/docs/cli/reference/index.md

@@ -32,16 +32,18 @@ spice [command] [--help]
 | [install](reference/install)       | Install or reinstall the Spice.ai runtime                              |
 | [login](reference/login)           | Login to Spice.ai or configure credentials for data sources            |
 | [models](reference/models)         | Lists models loaded by the Spice runtime                               |
-| nsql                               | Text-to-SQL REPL - translate natural language to SQL                   |
+| [nsql](reference/nsql)             | Text-to-SQL REPL - translate natural language to SQL                   |
 | [pods](reference/pods)             | Lists Spicepods loaded by the Spice runtime                            |
 | [query](reference/query)           | Submit an async query or start an interactive async query REPL         |
 | [refresh](reference/refresh)       | Refresh a dataset loaded by the Spice runtime                          |
 | [run](reference/run)               | Run Spice - starts the Spice runtime, installing if necessary          |
 | [search](reference/search)         | Search datasets with embeddings                                        |
 | [sql](reference/sql)               | Start an interactive SQL query session against the Spice runtime       |
+| [spiced](reference/spiced)         | Spice runtime binary — direct invocation reference                     |
 | [status](reference/status)         | Spice runtime status                                                   |
 | [trace](reference/trace)           | Return traces for operations that occurred in Spice                    |
 | [upgrade](reference/upgrade)       | Upgrades the Spice CLI and runtime to the latest release               |
+| [validate](reference/validate)     | Validate a spicepod.yaml without starting the runtime                  |
 | [version](reference/version)       | Spice CLI version                                                      |
 | workers                            | Lists workers loaded by the Spice runtime                              |
 

website/docs/cli/reference/init.md

@@ -23,13 +23,21 @@ spice init [app_name]
 ```
 > spice init taxi-trips
 
-2025/01/09 16:43:02 INFO Initialized taxi-trips/spicepod.yaml
+Initialized taxi-trips/spicepod.yaml
+
+Next steps:
+  cd taxi-trips
+  spice dataset configure   # add a dataset interactively
+  spice run                 # start the runtime
+
+Docs: https://spiceai.org/docs/
 ```
 
-The command creates a `spicepod.yaml` with basic initial metadata about the app. For this example, the `spicepod.yaml` file is initialized to the following:
+The command creates a `spicepod.yaml` with basic initial metadata about the app, including a `yaml-language-server` schema directive for editor support (VS Code, Neovim, IntelliJ). For this example, the `spicepod.yaml` file is initialized to the following:
 ```yaml
 # File: ./taxi-trips/spicepod.yaml
 
+# yaml-language-server: $schema=https://raw.githubusercontent.com/spiceai/spiceai/trunk/.schema/spicepod.schema.json
 version: v2
 kind: Spicepod
 name: taxi-trips
@@ -40,13 +48,20 @@ If no app name is provided, Spice initializes the Spicepod in the current workin
 > spice init
 
 name: (taxi-trips)? 
-2025/01/09 16:48:37 INFO Initialized spicepod.yaml
+Initialized spicepod.yaml
+
+Next steps:
+  spice dataset configure   # add a dataset interactively
+  spice run                 # start the runtime
+
+Docs: https://spiceai.org/docs/
 ```
 
 After execution, the current working directory contains the file `spicepod.yaml` with the same configuration as the previous example:
 ```yaml
 # File: ./spicepod.yaml
 
+# yaml-language-server: $schema=https://raw.githubusercontent.com/spiceai/spiceai/trunk/.schema/spicepod.schema.json
 version: v2
 kind: Spicepod
 name: taxi-trips

website/docs/cli/reference/nsql.md

@@ -0,0 +1,97 @@
+---
+title: 'nsql'
+sidebar_label: 'nsql'
+pagination_prev: null
+pagination_next: null
+description: 'spice nsql CLI documentation'
+---
+
+Text-to-SQL REPL — translate natural language queries into SQL using a model loaded by the Spice runtime. The REPL sends the input to the runtime's `/v1/nsql` endpoint and prints the generated SQL along with the executed results.
+
+## Requirements
+
+- Spice runtime must be running
+- At least one [model](../../components/models/) defined in `spicepod.yaml` and ready
+
+## Usage
+
+```shell
+spice nsql [flags]
+spice nsql [flags] [command]
+```
+
+### Flags
+
+- `--model`, `-m <model>` Target model for Text-to-SQL generation. When omitted, the CLI uses the single ready model or prompts for a choice if several models are ready.
+- `-h`, `--help` Print usage information.
+
+### Subcommands
+
+- [`analyze`](#analyze) Analyze Text-to-SQL performance by comparing the generated SQL against an expected SQL query.
+
+## Examples
+
+Start an interactive Text-to-SQL session:
+
+```shell
+$ spice nsql
+Welcome to the Spice.ai NSQL REPL!
+
+Using model:
+ openai
+
+Enter a query in natural language.
+nsql> show the top 5 longest taxi trips
+```
+
+Pass `--model` to select a specific model when more than one is ready:
+
+```shell
+spice nsql --model openai
+```
+
+Type `exit`, `quit`, `.exit`, or `.quit` — or press `Ctrl+D` — to leave the REPL. Inputs are saved to `nsql_history.txt` for recall with the up-arrow key.
+
+## analyze
+
+The `spice nsql analyze` subcommand evaluates Text-to-SQL quality by comparing a generated SQL query against an expected SQL query and reporting accuracy and performance metrics.
+
+### Usage
+
+```shell
+spice nsql analyze --query <natural-language-query> --expected <expected-sql> --model <model>
+```
+
+### Flags
+
+- `--query <query>` Natural language query to analyze. Required.
+- `--expected <sql>` Expected SQL query to compare the generated SQL against. Required.
+- `--model`, `-m <model>` Model to use for Text-to-SQL. Required.
+
+### Metrics
+
+Functional metrics (generated vs. expected SQL):
+
+- `exact_match` — `1.0` if the generated SQL exactly matches the expected SQL, `0.0` otherwise.
+- `correct_tables` — Intersection-over-Union (IoU) of tables referenced.
+- `correct_projections` — IoU of projected columns/expressions.
+- `correct_schema` — IoU of output schema fields.
+
+Performance metrics (read from `runtime.task_history` via the request's W3C trace ID):
+
+- `input_tokens` — Total prompt tokens used by LLM calls.
+- `output_tokens` — Total completion tokens generated by the LLM.
+- `latency_ms` — End-to-end latency of the nsql request.
+- `sql_duration_ms` — Total time spent executing SQL queries.
+- `llm_duration_ms` — Total time spent in LLM inference.
+- `sql_query_count` — Number of SQL queries executed.
+- `llm_count` — Number of LLM completion calls made.
+
+### Example
+
+```shell
+spice nsql analyze \
+  --model openai \
+  --query "how many taxi trips are there?" \
+  --expected "SELECT COUNT(*) FROM taxi_trips"
+```

website/docs/cli/reference/run.md

@@ -7,6 +7,8 @@ pagination_next: null
 
 Run Spice - starts the Spice runtime, installing if necessary.
 
+`spice run` is a wrapper around the [`spiced`](./spiced) runtime binary. It installs `spiced` on first use, applies developer-friendly defaults, and forwards arguments after `--` to the runtime. To invoke the runtime directly (for containers, systemd units, or CI), see the [`spiced` reference](./spiced).
+
 ### Usage
 
 ```shell

website/docs/cli/reference/spiced.md

@@ -0,0 +1,122 @@
+---
+title: 'spiced'
+sidebar_label: 'spiced'
+description: 'Command-line reference for the spiced runtime binary — flags, defaults, and differences from spice run.'
+keywords: [spice.ai, cli, spiced, runtime, binary, flags]
+pagination_prev: null
+pagination_next: null
+---
+
+`spiced` is the Spice.ai runtime binary. It hosts the HTTP and Flight servers, loads a Spicepod, and serves queries.
+
+Most users invoke `spiced` indirectly via [`spice run`](./run), which installs the runtime, applies developer-friendly defaults, and forwards CLI arguments. This page documents `spiced` itself for operators who run the binary directly (for example in containers, systemd units, or CI).
+
+### Usage
+
+```shell
+spiced [flags] [SPICEPOD_PATH]
+```
+
+If `SPICEPOD_PATH` is omitted, `spiced` uses the current working directory.
+
+### Runtime flags
+
+- `--http <BIND_ADDRESS>` — HTTP server bind address. Default: `127.0.0.1:8090`.
+- `--flight <FLIGHT_BIND_ADDRESS>` — Arrow Flight server bind address. Default: `127.0.0.1:50051`.
+- `--metrics <BIND_ADDRESS>` — Enable the Prometheus scrape endpoint on the given address. Disabled by default.
+- `--pods-watcher-enabled` — Watch the Spicepod directory for changes and hot-reload. Disabled by default.
+- `--telemetry-enabled <true|false>` — Override anonymous telemetry collection. When unset, the value is taken from `runtime.telemetry.enabled` in the Spicepod.
+- `--version` — Print the runtime version and exit.
+- `-v`, `--verbose` — Increase log verbosity. Repeat (`-vv`) for debug output.
+- `--very-verbose` — Equivalent to `-vv`.
+- `--set-runtime <key.subkey=value>` — Override a runtime configuration value. Can be repeated; see [examples in `spice run`](./run#--set-runtime).
+
+### TLS flags
+
+These configure TLS for the HTTP and Flight servers.
+
+- `--tls-enabled` — Enable TLS. Default: `false`.
+- `--tls-certificate <PEM>` — TLS certificate as an inline PEM string.
+- `--tls-certificate-file <PATH>` — Path to a PEM-encoded certificate file.
+- `--tls-key <PEM>` — TLS private key as an inline PEM string.
+- `--tls-key-file <PATH>` — Path to a PEM-encoded private key file.
+
+### Cluster flags
+
+Used when running `spiced` as part of a [distributed cluster](../../features/distributed-query). Omit these for standalone operation.
+
+- `--role <scheduler|executor>` — Explicit cluster role. If omitted but `--scheduler-address` is set, the role defaults to `executor`.
+- `--scheduler-address <URL>` — URL of the scheduler service. Required on executors.
+- `--node-bind-address <BIND_ADDRESS>` — Bind address for the internal cluster gRPC service. Default: `0.0.0.0:50052`.
+- `--node-advertise-address <HOST_OR_IP>` — Hostname or IP this node advertises to the rest of the cluster.
+- `--node-mtls-ca-certificate-file <PATH>` — CA certificate used to validate peer node identities.
+- `--node-mtls-certificate-file <PATH>` — Certificate file used for both server TLS and client mTLS on the node port.
+- `--node-mtls-key-file <PATH>` — Private key file for the node certificate.
+- `--allow-insecure-connections` — Allow cluster communication without mTLS. Use only in development or testing environments.
+
+### SQL REPL flags
+
+- `--repl` — Start a SQL REPL against the runtime's Flight endpoint instead of serving requests.
+- `--repl-flight-endpoint <HTTP_ENDPOINT>` — Flight endpoint the REPL connects to. Default: `http://localhost:50051`.
+- `--http-endpoint <HTTP_ENDPOINT>` — HTTP endpoint the REPL connects to. Default: `http://localhost:8090`.
+- `--tls-root-certificate-file <PATH>` — Root certificate used to verify the runtime's TLS certificate.
+- `--client-tls-certificate-file <PATH>` — Client certificate for mTLS authentication. Must be used with `--client-tls-key-file`.
+
+### Differences from `spice run`
+
+`spice run` is a thin launcher around `spiced`. When it spawns the runtime, it adds a few flags automatically that `spiced` does **not** apply when invoked directly:
+
+| Behavior                    | `spice run`                                                 | `spiced`                                            |
+| --------------------------- | ----------------------------------------------------------- | --------------------------------------------------- |
+| Runtime installation        | Auto-installs `spiced` if missing                           | Must already be installed                           |
+| Pods watcher                | Enabled (`--pods-watcher-enabled`)                          | Disabled by default                                 |
+| Task history captured output | Forced to `truncated` via `--set-runtime`                   | Uses the Spicepod value (default: `full`)           |
+| `--endpoint` scheme routing | Supported — `http://…` sets HTTP, `grpc://…` sets Flight    | Not supported — use `--http` and `--flight` directly |
+
+Operators running `spiced` directly who want `spice run`-equivalent behavior should pass `--pods-watcher-enabled` and, if desired, `--set-runtime task_history.captured_output=truncated`.
+
+### Examples
+
+#### Start the runtime with a Spicepod in the current directory
+
+```shell
+spiced
+```
+
+#### Bind the HTTP and Flight servers to all interfaces
+
+```shell
+spiced --http 0.0.0.0:8090 --flight 0.0.0.0:50051
+```
+
+#### Enable Prometheus metrics
+
+```shell
+spiced --metrics 0.0.0.0:9090
+```
+
+#### Enable TLS
+
+```shell
+spiced --tls-enabled \
+  --tls-certificate-file /path/to/cert.pem \
+  --tls-key-file /path/to/key.pem
+```
+
+#### Run as a cluster executor
+
+```shell
+spiced --role executor \
+  --scheduler-address https://scheduler.example.internal:50052 \
+  --node-advertise-address executor-1.example.internal \
+  --node-mtls-ca-certificate-file /etc/spice/ca.pem \
+  --node-mtls-certificate-file /etc/spice/node.pem \
+  --node-mtls-key-file /etc/spice/node.key
+```
+
+#### Override runtime configuration at launch
+
+```shell
+spiced --set-runtime task_history.captured_output=none \
+       --set-runtime results_cache.enabled=false
+```