feat(cli)!: add agent-first chat output contract

Author: coconut-yc

cli/AGENTS.md

@@ -77,11 +77,13 @@ token (emitting as a JSON array is planned for v0.8).
 `detail` carries structured per-error context (e.g. `unknown_subcommand`'s
 `available[]` list).
 
-### NDJSON event stream (chat / session ask)
+### Buffered JSON and NDJSON streams (chat / session ask)
 
-`--format json` and `--format ndjson` both produce one JSON event per line —
-no envelope wrapping. The CLI injects exactly one event (`init`) at the head;
-all subsequent events pass through verbatim from the SDK:
+`--format json` (the default) buffers the SSE stream and emits one normal
+success envelope containing the final answer, slim references, session
+pointers, and optional thinking (`--verbose`). `--format ndjson` is the raw
+event/debug surface: the CLI injects exactly one `init` event at the head and
+passes all subsequent SDK events through verbatim:
 
 ```
 {"type":"init","session_id":"...","kb_id":"...","profile":"...","agent_id":"..."}
@@ -170,7 +172,7 @@ is or isn't aligned with.
 
 | | |
 |---|---|
-| **WeKnora** | streaming commands (`chat`, `session ask`) emit bare `{type:...}` per line; no envelope |
+| **WeKnora** | `chat` / `session ask --format ndjson` emit bare `{type:...}` per line; default JSON buffers the stream into one envelope |
 | **Rationale** | This matches established practice across NDJSON-emitting CLIs and webhook protocols. A streaming envelope requires unwrap before dispatch — net burden with no benefit. |
 
 ### 5. No `schema_version` field in payload

cli/CHANGELOG.md

@@ -12,6 +12,28 @@ CLI history before v0.3 is recorded in the project root
 
 ## [Unreleased]
 
+### Breaking
+- `chat` and `session ask` now distinguish JSON from NDJSON: the default
+  `--format json` buffers the stream into one `{ok:true,data:{...}}` envelope;
+  use `--format ndjson` for the previous raw event stream.
+- Buffered JSON and MCP chat/session references clear full chunk `content`;
+  fetch the passage on demand with `chunk view <parent_chunk_id>` (or the
+  reference `id` when no parent is present).
+- `chat` / `session ask --format text` hide thinking and reflection by default;
+  pass `--verbose` to include the reasoning trace.
+
+### Added
+- `chat` / `session ask --verbose` includes thinking and reflection in buffered
+  JSON and rendered text output.
+- Buffered chat/session errors include the auto-created `session_id` in
+  `error.detail` so interrupted sessions remain recoverable.
+
+### Changed
+- Buffered JSON/MCP references omit full chunk content while preserving chunk
+  identifiers and citation metadata; fetch full passages with `chunk view`.
+- NDJSON remains an unmodified SDK event trace, including reasoning and full
+  reference payloads.
+
 ### Fixed
 - Streaming SDK calls are no longer cut off by the client's default 30-second
   timeout (explicit `WithTimeout` values remain honored), and SSE data lines

cli/cmd/chat/chat.go

@@ -1,21 +1,24 @@
 // Package chat implements `weknora chat <text>` - the streaming RAG answer
 // entry point.
 //
-// Two output modes share a single SDK call:
+// Three output modes share a single SDK call:
 //
-//   - Stream mode (TTY + --format text): write each StreamResponse.Content
-//     fragment directly to iostreams.IO.Out as it arrives, then print a
-//     footer with knowledge references. This is the "feels alive" UX a
-//     human typing in a terminal expects.
+//   - JSON mode (--format json, the default): accumulate the whole stream,
+//     then emit one success envelope {ok, data:{answer, references,
+//     session_id}}. The agent-recommended shape — parse one object
+//     instead of reassembling an event stream.
 //
-//   - NDJSON mode (--format json / --format ndjson / pipe): inject a CLI
-//     "init" event at stream head, then pass through every SDK event verbatim
-//     as NDJSON lines. Agents and pipes get a live event stream they can
-//     parse incrementally. --format json routes here too — buffered JSON
-//     envelope makes no sense for a streaming command.
+//   - Text mode (--format text): write each StreamResponse.Content fragment
+//     directly to iostreams.IO.Out as it arrives (TTY), then print a
+//     references footer. The "feels alive" UX a human typing in a terminal
+//     expects.
+//
+//   - NDJSON mode (--format ndjson): inject a CLI "init" event at stream
+//     head, then pass through every SDK event verbatim as NDJSON lines —
+//     the raw protocol trace, for debugging / advanced consumers.
 //
 // The SDK's KnowledgeQAStream callback contract is invoked sequentially on
-// one goroutine, so neither mode needs locking. The runChat core takes a
+// one goroutine, so no mode needs locking. The runChat core takes a
 // ChatService interface so tests inject a fake without standing up a real
 // SSE server.
 package chat
@@ -35,19 +38,22 @@ import (
 	sdk "github.com/Tencent/WeKnora/client"
 )
 
-// chatFields enumerates the NDJSON init-event fields surfaced for
-// `--format json` / `--format ndjson` discovery on `chat`. Reflects the
-// InitEvent head line + the raw SDK event vocabulary.
+// chatFields enumerates the fields surfaced for `--jq` projection discovery
+// on `chat`: the json object's data fields + the raw SDK event vocabulary
+// used by --format ndjson.
 var chatFields = []string{
-	"session_id", "kb_id",
-	// SDK event fields (pass-through): response_type, content, done,
-	// knowledge_references, assistant_message_id, session_id
+	"answer", "references", "thinking", "session_id", "assistant_message_id",
+	// NDJSON init + SDK event fields.
+	"type", "kb_id", "profile", "response_type", "content", "done", "knowledge_references", "data",
 }
 
 type Options struct {
 	Query     string
 	KBID      string
 	SessionID string
+	// Verbose surfaces the model's thinking/retrieval process in JSON/text.
+	// NDJSON is always raw and is unaffected by presentation flags.
+	Verbose bool
 }
 
 // ChatService is the narrow SDK surface this command depends on. *sdk.Client
@@ -69,12 +75,15 @@ answer back. By default a fresh session is created on first invocation; pass
 --session to continue an existing conversation.
 
 Modes:
+  --format json (default):       one JSON object {ok, data:{answer, references,
+                                 session_id}} once the stream completes —
+                                 parse one object, no event reassembly. The
+                                 agent-recommended shape.
   --format text:                 live token streaming + reference footer
-  --format json / --format ndjson / pipe (default): NDJSON event stream —
-                                 one init line at head (session_id, kb_id),
-                                 then raw SDK events verbatim. Both json
-                                 and ndjson flags produce the same NDJSON
-                                 stream.`,
+  --format ndjson:               raw NDJSON event stream — init line (session_id,
+                                 kb_id) then SDK events verbatim. Debug / advanced.
+
+Pass --verbose to include the model's thinking/reflection in JSON/text output.`,
 		Example: `  weknora chat "What is RRF?" --kb a32a63ff-fb36-4874-bcaa-30f48570a694
   weknora chat "Summarise this design doc" --kb my-kb --format json
   weknora chat "Continue?" --session sess_abc`,
@@ -103,12 +112,16 @@ Modes:
 	}
 	cmdutil.AddKBFlag(cmd)
 	cmd.Flags().StringVar(&opts.SessionID, "session", "", "Continue an existing chat session (skip auto-create)")
+	cmd.Flags().BoolVar(&opts.Verbose, "verbose", false, "Include the model's thinking/retrieval process in JSON/text output")
 	cmdutil.AddFormatFlag(cmd, chatFields...)
 	cmdutil.SetAgentHelp(cmd, cmdutil.AgentHelp{
-		UsedFor:       "Ask a streaming RAG question against a knowledge base. Produces an NDJSON event stream: init line (session_id, kb_id) then raw SDK events. Use --format json or --format ndjson.",
+		UsedFor:       "Ask a RAG question against a knowledge base. Default (--format json) returns ONE JSON object {ok, data:{answer, references, session_id}} after the stream completes — parse one object, no event reassembly. --format ndjson streams raw SDK events; --format text streams live tokens. --verbose adds the model's thinking to JSON/text.",
 		RequiredFlags: []string{"--kb"},
-		Examples:      []string{`weknora chat "What is RRF?" --kb kb_abc --format json`},
-		Output:        "NDJSON stream: {type:init, session_id, kb_id} then SDK events (response_type, content, done, knowledge_references, ...)",
+		Examples: []string{
+			`weknora chat "What is RRF?" --kb kb_abc`,
+			`weknora chat "What is RRF?" --kb kb_abc --jq '.data.answer'`,
+		},
+		Output: "Default --format json: {ok, data:{answer, references[], session_id, thinking?}}. --format ndjson: {type:init, session_id, kb_id} then SDK events (response_type, content, done, knowledge_references, ...).",
 	})
 	return cmd
 }
@@ -128,11 +141,8 @@ func runChat(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, s
 		return cmdutil.NewError(cmdutil.CodeServerError, "chat: no SDK client available")
 	}
 
-	// Streaming commands route --format json AND --format ndjson to the
-	// NDJSON event-stream path. A buffered envelope makes no sense for a
-	// streaming command. Only --format text uses the live renderer.
-	ndjsonMode := fopts != nil && (fopts.Mode == cmdutil.FormatJSON || fopts.Mode == cmdutil.FormatNDJSON)
-
+	// --format selects the output shape: json (default) accumulates and
+	// emits one object, ndjson streams raw SDK events, text renders live.
 	sessionID := opts.SessionID
 	autoCreated := false
 	if sessionID == "" {
@@ -156,24 +166,26 @@ func runChat(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, s
 		autoCreated = true
 	}
 
-	if ndjsonMode {
+	if fopts != nil && fopts.Mode == cmdutil.FormatNDJSON {
 		return runChatNDJSON(ctx, opts, sessionID, svc)
 	}
+	if fopts != nil && fopts.Mode == cmdutil.FormatJSON {
+		return runChatJSON(ctx, opts, fopts, sessionID, svc)
+	}
 
 	// Surface the auto-created session ID up-front so a user who hits ^C
 	// mid-stream still has the pointer to resume - no need to scroll back
-	// past tokens. Skipped in NDJSON mode (it appears in the init event).
+	// past tokens. Skipped in json/ndjson mode (session_id is in the output).
 	if autoCreated {
 		fmt.Fprintf(iostreams.IO.Err, "session: %s (use --session to continue)\n", sessionID)
 	}
 
 	return runChatText(ctx, opts, sessionID, autoCreated, svc)
 }
 
-// runChatNDJSON handles --format json and --format ndjson paths.
-// Emits a CLI init event at stream head, then passes every SDK event through
-// verbatim as NDJSON lines. No buffering — callers parse the stream
-// incrementally.
+// runChatNDJSON handles the --format ndjson path: emits a CLI init event at
+// stream head, then passes every SDK event through verbatim as NDJSON lines.
+// No buffering — callers parse the stream incrementally.
 func runChatNDJSON(ctx context.Context, opts *Options, sessionID string, svc ChatService) error {
 	w := iostreams.IO.Out
 
@@ -198,6 +210,8 @@ func runChatNDJSON(ctx context.Context, opts *Options, sessionID string, svc Cha
 		Channel:          "api",
 	}
 	cb := func(r *sdk.StreamResponse) error {
+		// NDJSON is the raw protocol/debug surface: do not filter events or
+		// mutate their payloads. JSON/text modes own presentation filtering.
 		return output.EmitSDKEvent(w, r)
 	}
 	if err := svc.KnowledgeQAStream(ctx, sessionID, req, cb); err != nil {
@@ -227,9 +241,14 @@ func runChatText(ctx context.Context, opts *Options, sessionID string, autoCreat
 
 	cb := func(r *sdk.StreamResponse) error {
 		if streamMode && r != nil && r.Content != "" {
-			// Best-effort write; if stdout dies the SDK will surface the
-			// error on the next iteration. No need to bail early.
-			_, _ = iostreams.IO.Out.Write([]byte(r.Content))
+			// Default hides the reasoning pass (thinking/reflection); --verbose
+			// streams it inline with the answer.
+			isReasoning := r.ResponseType == sdk.ResponseTypeThinking || r.ResponseType == sdk.ResponseTypeReflection
+			if !isReasoning || opts.Verbose {
+				// Best-effort write; if stdout dies the SDK will surface the
+				// error on the next iteration. No need to bail early.
+				_, _ = iostreams.IO.Out.Write([]byte(r.Content))
+			}
 		}
 		acc.Append(r)
 		return nil
@@ -281,14 +300,96 @@ func runChatText(ctx context.Context, opts *Options, sessionID string, autoCreat
 			fmt.Fprintln(out)
 		}
 	} else {
-		fmt.Fprint(out, answer)
-		if !strings.HasSuffix(answer, "\n") {
+		rendered := answer
+		if opts.Verbose {
+			rendered = acc.Thinking() + rendered
+		}
+		fmt.Fprint(out, rendered)
+		if !strings.HasSuffix(rendered, "\n") {
 			fmt.Fprintln(out)
 		}
 	}
 	format.WriteReferences(out, references)
 	return nil
 }
 
+// runChatJSON handles the --format json path (the default): accumulate the
+// full stream with no live output, then emit a single success envelope —
+// one object an agent parses instead of an NDJSON event stream to reassemble.
+// The envelope carries the answer, slim references (Content stripped — fetch
+// full passages via `chunk view <parent_chunk_id>`), optional thinking
+// (--verbose), and session pointers.
+func runChatJSON(ctx context.Context, opts *Options, fopts *cmdutil.FormatOptions, sessionID string, svc ChatService) error {
+	req := &sdk.KnowledgeQARequest{
+		Query:            opts.Query,
+		KnowledgeBaseIDs: []string{opts.KBID},
+		AgentEnabled:     false,
+		WebSearchEnabled: false,
+		Channel:          "api",
+	}
+
+	acc := &sse.Accumulator{}
+	cb := func(r *sdk.StreamResponse) error {
+		acc.Append(r)
+		return nil
+	}
+
+	if err := svc.KnowledgeQAStream(ctx, sessionID, req, cb); err != nil {
+		if cmdutil.IsCancelled(ctx, err) {
+			return chatStreamError(cmdutil.Wrapf(cmdutil.CodeOperationCancelled, err, "chat cancelled"), sessionID, acc.AssistantMessageID)
+		}
+		return chatStreamError(cmdutil.WrapHTTP(err, "knowledge qa stream"), sessionID, acc.AssistantMessageID)
+	}
+	if !acc.Done() {
+		return chatStreamError(
+			cmdutil.NewError(cmdutil.CodeSSEStreamAborted, "stream ended without a terminal event"),
+			sessionID,
+			acc.AssistantMessageID,
+		)
+	}
+
+	sid := acc.SessionID
+	if sid == "" {
+		sid = sessionID
+	}
+	data := chatResult{
+		Answer:             acc.Result(),
+		References:         acc.References,
+		SessionID:          sid,
+		AssistantMessageID: acc.AssistantMessageID,
+	}
+	if opts.Verbose {
+		data.Thinking = acc.Thinking()
+	}
+
+	// Reaching here means fopts.Mode is FormatJSON (the only caller). Route
+	// through FormatOptions.Emit so --jq projection and the success-envelope
+	// contract apply, matching every other non-streaming command. A nil fopts
+	// (direct-test entry) defaults to JSON so the object still emits.
+	if fopts == nil {
+		fopts = &cmdutil.FormatOptions{Mode: cmdutil.FormatJSON}
+	}
+	return fopts.Emit(iostreams.IO.Out, data, nil)
+}
+
+func chatStreamError(err *cmdutil.Error, sessionID, assistantMessageID string) *cmdutil.Error {
+	detail := map[string]any{"session_id": sessionID}
+	if assistantMessageID != "" {
+		detail["assistant_message_id"] = assistantMessageID
+	}
+	return err.WithDetail(detail)
+}
+
+// chatResult is the --format json data payload: one object an agent parses
+// instead of an NDJSON stream. References are slim (Content stripped by the
+// accumulator). Wrapped by Emit as {ok:true, data:<chatResult>, meta}.
+type chatResult struct {
+	Answer             string              `json:"answer"`
+	References         []*sdk.SearchResult `json:"references,omitempty"`
+	Thinking           string              `json:"thinking,omitempty"`
+	SessionID          string              `json:"session_id"`
+	AssistantMessageID string              `json:"assistant_message_id,omitempty"`
+}
+
 // compile-time check: the production SDK client implements ChatService.
 var _ ChatService = (*sdk.Client)(nil)