Feat: Add support for JSON endpoints in Conversational Search (#690)

* Feat: Add support for JSON endpoints in Conversational Search
ref: #683


---------

Co-authored-by: djklim87 <klim@manticoresearch.com>

Author: KlimTodrik

composer.lock

@@ -102,7 +102,7 @@ private static function createModel(
 	): TaskResult {
 		/** @var array{identifier: string, model: string, description?: string,
 		 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-		 *   max_document_length?: string|int} $config
+		 *   max_document_length?: string|int, custom_prompt?: string} $config
 		 */
 		$config = $payload->params;
 		$createConfig = (new ModelConfigValidator())->validate($config);
@@ -286,8 +286,7 @@ private static function handleConversation(
 			);
 		}
 
-		$responseWithRefs = $response['content'];
-		$responseText = self::stripReferences($responseWithRefs);
+		$responseText = $response['content'];
 		$tokensUsed = $response['metadata']['tokens_used'];
 
 		$turn = new ConversationTurn(
@@ -307,14 +306,12 @@ private static function handleConversation(
 				'user_query' => $request->query,
 				'search_query' => $queries['search_query'],
 				'response' => $responseText,
-				'response_with_refs' => $responseWithRefs,
 				'sources' => json_encode($searchResults),
 			]
 		)->column('conversation_uuid', Column::String)
 			->column('user_query', Column::String)
 			->column('search_query', Column::String)
 			->column('response', Column::String)
-			->column('response_with_refs', Column::String)
 			->column('sources', Column::String);
 	}
 
@@ -532,16 +529,6 @@ private static function logPreprocessingResults(
 		Buddy::debugv("Chat: └─ Exclude query: '{$queries['exclude_query']}'");
 	}
 
-	/**
-	 * @return string
-	 */
-	private static function stripReferences(string $responseWithRefs): string {
-		$response = preg_replace('/\s*\[ref:[^\]\s]+\]/', '', $responseWithRefs);
-		$response = preg_replace('/[ 	]+([.,;:!?])/', '$1', (string)$response);
-
-		return trim((string)$response);
-	}
-
 	/**
 	 * @param array<int, array<string, mixed>> $searchResults
 	 * @param string $contentFields
@@ -616,7 +603,13 @@ private static function generateResponse(
 	): array {
 		$provider->configure($model);
 
-		$prompt = self::buildPrompt($query, $context, $history->payload());
+		$customPrompt = $model['settings']['custom_prompt'] ?? null;
+		$prompt = self::buildPrompt(
+			$query,
+			$context,
+			$history->payload(),
+			is_string($customPrompt) ? $customPrompt : ''
+		);
 		$settings = self::getLlmRequestOptions();
 
 		return $provider->generateResponse($prompt, $settings);
@@ -640,26 +633,31 @@ private static function getLlmRequestOptions(): array {
 	 *
 	 * @throws JsonException
 	 */
-	private static function buildPrompt(string $query, string $context, array $history): string {
+	private static function buildPrompt(
+		string $query,
+		string $context,
+		array $history,
+		string $customPrompt = ''
+	): string {
 		$historyJson = (string)json_encode($history, JSON_THROW_ON_ERROR);
-
-		return "system:\n"
-			. "You are a context-only answer writer.\n\n"
-			. 'Answer using only the provided context. Do not use outside knowledge, memory, assumptions, '
-			. "or unsupported facts.\n"
-			. 'Keep the answer concise and under ' . self::RESPONSE_MAX_TOKENS . " tokens.\n"
-			. "Citation rule:\n\n"
-			. "First write the answer with no citations at all.\n"
-			. "After the answer is finished, append the reference context ID (context[].id) once.\n"
-			. "If id reference was used, it must be exactly: [ref:<id>]\n"
-			. "Never put a reference ID after individual sentences.\n"
-			. "Never repeat the same reference ID.\n\n"
-			. "If the context is insufficient, answer exactly:\n"
-			. "I don’t have enough information in the provided context to answer.\n\n"
-			. "user:\n"
-			. "Query: $query\n\n"
-			. "History:\n```json\n$historyJson\n```\n"
-			. "Context:\n```json\n$context\n```";
+		$prompt = trim($customPrompt) !== ''
+			? trim($customPrompt)
+			: 'Respond conversationally. Response should be based ONLY on the provided history and context sections' .
+				"(IMPORTANT !!! You can't use your own knowledge to add anything that isn't mentioned there). " .
+				'Do not exceed the response token limit (' .
+				self::RESPONSE_MAX_TOKENS .
+				'), and end the answer cleanly before reaching it.';
+
+		return $prompt . ' ' .
+			'<main>' .
+			"<history>\n" .
+			"```json\n" .
+			$historyJson .
+			"\n```\n" .
+			"</history>\n" .
+			"<context>$context</context>\n" .
+			"<query>$query</query>\n" .
+			'</main>';
 	}
 
 	/**

src/Plugin/ConversationalSearch/Handler.php

@@ -14,6 +14,8 @@
 use Manticoresearch\Buddy\Core\Error\QueryParseError;
 
 final class ModelConfigValidator {
+	private const int MAX_CUSTOM_PROMPT_BYTES = 32768;
+
 	/**
 	 * Supported flat fields for CREATE CHAT MODEL.
 	 *
@@ -27,16 +29,17 @@ final class ModelConfigValidator {
 		'timeout',
 		'retrieval_limit',
 		'max_document_length',
+		'custom_prompt',
 	];
 
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return array{name: string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int}
+	 *   max_document_length?: string|int, custom_prompt?: string}
 	 * @throws QueryParseError
 	 */
 	public function validate(array $config): array {
@@ -46,6 +49,7 @@ public function validate(array $config): array {
 		$this->validateTimeout($config);
 		$this->validateRetrievalLimit($config);
 		$this->validateMaxDocumentLength($config);
+		$this->validateCustomPrompt($config);
 
 		$createConfig = ['name' => $config['identifier']];
 		foreach (self::MODEL_FIELDS as $field) {
@@ -58,7 +62,7 @@ public function validate(array $config): array {
 
 		/** @var array{name: string, model: string, description?: string,
 		 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-		 *   max_document_length?: string|int} $createConfig
+		 *   max_document_length?: string|int, custom_prompt?: string} $createConfig
 		 */
 		return $createConfig;
 	}
@@ -85,7 +89,7 @@ private function validateSupportedFields(array $config): void {
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return void
 	 * @throws QueryParseError
@@ -99,7 +103,7 @@ private function validateRequiredFields(array $config): void {
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return void
 	 * @throws QueryParseError
@@ -115,7 +119,7 @@ private function validateModelId(array $config): void {
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return void
 	 * @throws QueryParseError
@@ -142,7 +146,7 @@ private function validateTimeout(array $config): void {
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return void
 	 * @throws QueryParseError
@@ -169,7 +173,7 @@ private function validateRetrievalLimit(array $config): void {
 	/**
 	 * @param array{identifier:string, model: string, description?: string,
 	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return void
 	 * @throws QueryParseError
@@ -198,4 +202,29 @@ private function validateMaxDocumentLength(array $config): void {
 			);
 		}
 	}
+
+	/**
+	 * @param array{identifier:string, model: string, description?: string,
+	 *   api_key?: string, base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
+	 *
+	 * @return void
+	 * @throws QueryParseError
+	 */
+	private function validateCustomPrompt(array $config): void {
+		if (!isset($config['custom_prompt'])) {
+			return;
+		}
+
+		$customPrompt = $config['custom_prompt'];
+		if (trim($customPrompt) === '') {
+			throw QueryParseError::create('custom_prompt must be a non-empty string');
+		}
+
+		if (strlen($customPrompt) > self::MAX_CUSTOM_PROMPT_BYTES) {
+			throw QueryParseError::create(
+				'custom_prompt must be at most ' . self::MAX_CUSTOM_PROMPT_BYTES . ' bytes'
+			);
+		}
+	}
 }

src/Plugin/ConversationalSearch/ModelConfigValidator.php

@@ -32,7 +32,7 @@ class ModelManager {
 	 * @param HTTPClient $client
 	 * @param array{name: string, model: string, description?: string, api_key?: string,
 	 *   base_url?: string, timeout?: string|int, retrieval_limit?: string|int,
-	 *   max_document_length?: string|int} $config
+	 *   max_document_length?: string|int, custom_prompt?: string} $config
 	 *
 	 * @return string Model name
 	 * @throws ManticoreSearchClientError|ManticoreSearchResponseError|QueryParseError

src/Plugin/ConversationalSearch/ModelManager.php

@@ -101,6 +101,7 @@ Model with prompt, transport options, and retrieval settings:
 ```sql
 CREATE CHAT MODEL support_assistant (
     model='openai:gpt-4o-mini',
+    custom_prompt='You are a support assistant. Answer using the retrieved context and mention source ids.',
     api_key='your-provider-api-key',
     base_url='http://host.docker.internal:8787/v1',
     timeout=60,
@@ -115,6 +116,7 @@ Common options:
 |---|---:|---|
 | `model` | Yes | LLM model id in `provider:model` format |
 | `description` | No | Stored description |
+| `custom_prompt` | No | Prompt instructions used to build the answer from the retrieved context and conversation history; if set, it must be non-empty and at most 32768 bytes |
 | `api_key` | No | Provider API key passed to the `llm` extension |
 | `base_url` | No | Provider or proxy base URL |
 | `timeout` | No | LLM request timeout, `1..65536` |
@@ -124,6 +126,12 @@ Common options:
 Model names in `CREATE CHAT MODEL` may contain letters, numbers, and
 underscores only.
 
+`custom_prompt` is optional. If it is omitted, Buddy uses its default answer
+prompt. If it is set, it must contain non-whitespace text and must not exceed
+32768 bytes. To make the LLM return source citations, include that instruction
+in `custom_prompt`, for example: `Cite sources as [ref:<id>] using source row
+ids from the context.`
+
 `model` is validated as `provider:model`, for example:
 
 ```sql
@@ -251,9 +259,8 @@ The legacy JSON field name `fields` is accepted as an alias, but requests must
 not include both `vector_field` and `fields`.
 
 HTTP JSON conversation responses use the same logical columns as `CALL CHAT`:
-`conversation_uuid`, `user_query`, `search_query`, `response`,
-`response_with_refs`, and `sources`. `sources` is currently returned as a JSON
-string containing the retrieved source rows.
+`conversation_uuid`, `user_query`, `search_query`, `response`, and `sources`.
+`sources` is currently returned as a JSON string containing the retrieved source rows.
 
 The table argument must be a plain table identifier, optionally qualified as
 `database.table`. The vector field argument must be a plain field identifier.
@@ -298,8 +305,7 @@ Behavior:
 | `conversation_uuid` | Existing or generated conversation id |
 | `user_query` | Original user query |
 | `search_query` | Standalone search query used for retrieval |
-| `response` | LLM answer with inline references like `[ref:<id>]` removed |
-| `response_with_refs` | Full LLM answer including inline source references like `[ref:<id>]`, where `<id>` is the source row id |
+| `response` | LLM answer as generated |
 | `sources` | JSON string containing retrieved source rows |
 
 Example response shape:
@@ -310,12 +316,13 @@ Example response shape:
   "user_query": "What is vector search?",
   "search_query": "vector search, embeddings, similarity search",
   "response": "Vector search finds similar items by comparing embeddings...",
-  "response_with_refs": "Vector search finds similar items by comparing embeddings [ref:1]...",
   "sources": "[{\"id\":1,\"title\":\"Vector Search\",\"content\":\"...\",\"knn_dist\":0.12}]"
 }
 ```
 
-Vector fields are intentionally absent from `sources`.
+Vector fields are intentionally absent from `sources`. If you want source citations,
+add citation instructions to `custom_prompt`; source row IDs are available in the
+context and `sources` payload.
 
 ## Model Management