Skip to content

Commit 8a5634c

Browse files
lichunnlichunn
authored
Fix：robot image issues and add test cases (#1815)
* fix：调整RobotChat逻辑和修复图片上传问题 * feat：添加测试用例 * fix:fix review
1 parent a79c9df commit 8a5634c

8 files changed

Lines changed: 1528 additions & 101 deletions

File tree

‎packages/plugins/robot/src/Main.vue‎

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@
1919
v-model:show="robotVisible"
2020
v-model:input="inputMessage"
2121
:status="mappedStatus"
22-
:chat-mode="robotSettingState.chatMode"
2322
:prompt-items="promptItems"
2423
:bubble-renderers="bubbleRenderers"
2524
:allowFiles="isVisualModel && robotSettingState.chatMode === ChatMode.Agent"
2625
:show-aborted="robotSettingState.chatMode !== ChatMode.Agent"
26+
:message-content-resolver="resolveChatMessageContent"
2727
:beforeSubmit="checkApiKey"
2828
:promptClickHandler="promptClickHandler"
2929
@fileSelected="handleFileSelected"
@@ -93,6 +93,16 @@ import { AgentRenderer } from './components/renderers'
9393
import useChat from './composables/useChat'
9494
import useModelConfig from './composables/core/useConfig'
9595
import { ChatMode } from './types/mode.types'
96+
import { STATUS } from './constants/status'
97+
import {
98+
AgentMessageStatus,
99+
RobotMessageContentType,
100+
RobotMessageRole,
101+
isAgentFinalStatus,
102+
type MessageResolverContext,
103+
type RobotMessage,
104+
type RobotRenderContentItem
105+
} from './types'
96106
import apiService from './services/api'
97107
98108
const props = defineProps({
@@ -106,6 +116,7 @@ const { robotSettingState, getModelCapabilities, updateThinkingState, getSelecte
106116
107117
const robotVisible = ref(false)
108118
const fullscreen = ref(false)
119+
const inputMessage = ref('')
109120
110121
watch(robotVisible, (visible) => {
111122
useLayout().layoutState.toolbars.render = visible ? META_APP.Robot : ''
@@ -150,7 +161,6 @@ const showSetting = ref(false)
150161
151162
const {
152163
mappedStatus,
153-
inputMessage,
154164
messages,
155165
changeChatMode,
156166
abortRequest,
@@ -224,9 +234,9 @@ const promptClickHandler = (item: PromptProps & { mode?: 'chat' | 'agent' }) =>
224234
changeChatMode(item.mode)
225235
}
226236
messages.value.push({
227-
role: 'user',
237+
role: RobotMessageRole.User,
228238
content: item.description || '',
229-
renderContent: [{ type: 'text', content: item.description }]
239+
renderContent: [{ type: RobotMessageContentType.Text, content: item.description }]
230240
})
231241
sendUserMessage()
232242
}
@@ -245,8 +255,67 @@ const openAIRobot = () => {
245255
useLayout().closeSetting(true)
246256
}
247257
248-
// 当前Robot的bubbleRenderers无法做到响应式更新，因此Agent模式的type要与Chat模式不同
249-
const bubbleRenderers = { 'agent-content': AgentRenderer, 'agent-loading': AgentRenderer }
258+
// 当前 Robot 的 bubbleRenderers 无法做到响应式更新，因此 Agent 模式需要独立的内容类型。
259+
// `agent-content` 表示 Agent 的最终内容片段；`agent-loading` 表示 Agent 处理中间态占位片段。
260+
const bubbleRenderers = {
261+
[RobotMessageContentType.AgentContent]: AgentRenderer,
262+
[RobotMessageContentType.AgentLoading]: AgentRenderer
263+
}
264+
265+
const resolveChatMessageContent = (message: RobotMessage, context: MessageResolverContext) => {
266+
const renderContent = Array.isArray(message.renderContent) ? message.renderContent : []
267+
const hasAgentContent = renderContent.some((item) => {
268+
return item.type === RobotMessageContentType.AgentContent || item.type === RobotMessageContentType.AgentLoading
269+
})
270+
const isAgentMessage = message.metadata?.chatMode === ChatMode.Agent || hasAgentContent
271+
272+
if (!isAgentMessage || message.role !== RobotMessageRole.Assistant) {
273+
return Array.isArray(message.renderContent) && message.renderContent.length > 0
274+
? message.renderContent
275+
: message.content
276+
}
277+
278+
// context.status 是当前整轮对话请求的运行状态，不是单条渲染片段状态。
279+
const isLastMessage = context.messages.at(-1) === message
280+
const isGenerating = Boolean(message.loading) || (isLastMessage && context.status !== STATUS.FINISHED)
281+
const resolvedRenderContent = isGenerating
282+
? renderContent
283+
: renderContent.filter((item) => item.type !== RobotMessageContentType.AgentLoading)
284+
const agentContents = resolvedRenderContent.filter(
285+
(item): item is RobotRenderContentItem =>
286+
item.type === RobotMessageContentType.AgentContent || item.type === RobotMessageContentType.AgentLoading
287+
)
288+
// item.status 是单个 agent 片段的业务结果，例如 success/failed/fix/loading。
289+
const finalStatus = agentContents.findLast((item) => isAgentFinalStatus(item.status))?.status
290+
291+
if (!Array.isArray(message.renderContent) || message.renderContent.length === 0) {
292+
const agentStatus = isAgentFinalStatus(message.metadata?.agentStatus)
293+
? message.metadata.agentStatus
294+
: AgentMessageStatus.Failed
295+
return [
296+
{
297+
type: RobotMessageContentType.AgentContent,
298+
status: agentStatus,
299+
content: message.content
300+
}
301+
]
302+
}
303+
304+
return resolvedRenderContent.map((item) => {
305+
if (item.type !== RobotMessageContentType.AgentContent || isGenerating) {
306+
return item
307+
}
308+
309+
if (!item.status || item.status === AgentMessageStatus.Loading) {
310+
return {
311+
...item,
312+
status: finalStatus || message.metadata?.agentStatus || AgentMessageStatus.Failed
313+
}
314+
}
315+
316+
return item
317+
})
318+
}
250319
251320
const handleFileSelected = async (formData: FormData, updateAttachment: (resourceUrl: string) => void) => {
252321
try {

‎packages/plugins/robot/src/components/chat/RobotChat.vue‎

Lines changed: 79 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</template>
1212

1313
<div class="robot-chat-container-content" ref="chatContainerRef">
14-
<div v-if="messages.filter((item) => item.role !== 'system').length === 0">
14+
<div v-if="messages.filter((item) => item.role !== RobotMessageRole.System).length === 0">
1515
<tr-welcome title="AI助手" description="您好，我是您的开发小助手" :icon="welcomeIcon" class="robot-welcome">
1616
</tr-welcome>
1717
<tr-prompts
@@ -99,10 +99,18 @@ import {
9999
type RawFileAttachment,
100100
type BubbleContentRendererMatch
101101
} from '@opentiny/tiny-robot'
102-
import { type ChatMessage } from '@opentiny/tiny-robot-kit'
103102
import { GeneratingStatus } from '../../constants/status'
104103
import { LoadingRenderer, MarkdownRenderer, ImgRenderer } from '../renderers'
105104
import { useNotify } from '@opentiny/tiny-engine-meta-register'
105+
import {
106+
RobotMessageContentType,
107+
RobotMessageRole,
108+
type MessageContentResolver,
109+
type RobotInputContentPart,
110+
type RobotMessage,
111+
type RobotRenderContentItem
112+
} from '../../types'
113+
import { extractMessageText } from '../../utils'
106114
107115
const props = defineProps({
108116
promptItems: {
@@ -113,7 +121,9 @@ const props = defineProps({
113121
type: Function
114122
},
115123
status: { type: String },
116-
chatMode: { type: String },
124+
messageContentResolver: {
125+
type: Function as PropType<MessageContentResolver>
126+
},
117127
allowFiles: {
118128
type: Boolean,
119129
default: false
@@ -139,7 +149,7 @@ const selectedAttachments = ref([])
139149
const robotVisible = defineModel<boolean>('show', { required: true })
140150
const fullscreen = defineModel<boolean>('fullscreen')
141151
const inputMessage = defineModel<string>('input', { required: true })
142-
const messages = defineModel<ChatMessage[]>('messages', { required: true })
152+
const messages = defineModel<RobotMessage[]>('messages', { required: true })
143153
const senderRef = ref<InstanceType<typeof TrSender> | null>(null)
144154
145155
watch(
@@ -159,63 +169,29 @@ const contentRendererMatches = computed<BubbleContentRendererMatch[]>(() => [
159169
},
160170
...Object.entries(props.bubbleRenderers).map(([type, renderer]) => ({
161171
priority: BubbleRendererMatchPriority.NORMAL,
162-
find: (_message: any, content: any) => content?.type === type,
172+
find: (_message: RobotMessage, content: RobotRenderContentItem) => content?.type === type,
163173
renderer
164174
})),
165175
{
166176
priority: BubbleRendererMatchPriority.NORMAL,
167-
find: (message: any, content: any) => content?.type === 'tool' && message.tool_calls?.length,
177+
find: (message: RobotMessage, content: RobotRenderContentItem) =>
178+
content?.type === RobotMessageContentType.Tool && Boolean(message.tool_calls?.length),
168179
renderer: BubbleRenderers.Tools
169180
},
170181
{
171182
priority: BubbleRendererMatchPriority.NORMAL,
172-
find: (message: any, content: any) =>
173-
!message.loading && message.content && (!content?.type || ['markdown', 'text'].includes(content.type)),
183+
find: (_message: RobotMessage, content: RobotRenderContentItem) =>
184+
!content?.type || [RobotMessageContentType.Markdown, RobotMessageContentType.Text].includes(content.type as any),
174185
renderer: MarkdownRenderer
175186
},
176187
{
177188
priority: BubbleRendererMatchPriority.NORMAL,
178-
find: (message: any) => message?.content?.[0]?.type === 'img' || message?.content?.[0]?.type === 'image',
189+
find: (_message: RobotMessage, content: RobotRenderContentItem) =>
190+
[RobotMessageContentType.Img, RobotMessageContentType.Image].includes(content?.type as any),
179191
renderer: ImgRenderer
180192
}
181193
])
182194
183-
const isAgentMessage = (message: any) => {
184-
const hasAgentContent = message.renderContent?.some((item: any) => {
185-
return item.type === 'agent-content' || item.type === 'agent-loading'
186-
})
187-
return message.metadata?.chatMode === 'agent' || hasAgentContent
188-
}
189-
190-
const resolveAgentRenderContent = (message: any) => {
191-
if (!isAgentMessage(message) || message.role !== 'assistant') {
192-
return message.renderContent
193-
}
194-
195-
const isLastMessage = messages.value.at(-1) === message
196-
const isGenerating = Boolean(message.loading) || (isLastMessage && GeneratingStatus.includes(props.status as any))
197-
const renderContent = isGenerating
198-
? message.renderContent
199-
: message.renderContent.filter((item: any) => item.type !== 'agent-loading')
200-
const agentContents = renderContent.filter((item: any) => item.type === 'agent-content')
201-
const finalStatus = agentContents.findLast((item: any) => ['success', 'failed', 'fix'].includes(item.status))?.status
202-
203-
return renderContent.map((item: any) => {
204-
if (item.type !== 'agent-content' || isGenerating) {
205-
return item
206-
}
207-
208-
if (!item.status || item.status === 'loading') {
209-
return {
210-
...item,
211-
status: finalStatus || message.metadata?.agentStatus || 'failed'
212-
}
213-
}
214-
215-
return item
216-
})
217-
}
218-
219195
// 处理文件选择事件
220196
const handleSingleFilesSelected = (files: File[] | null, retry = false) => {
221197
if (!files?.length) return
@@ -271,36 +247,56 @@ const getSvgIcon = (name: string, style?: CSSProperties) => {
271247
const aiAvatar = getSvgIcon('AI')
272248
const welcomeIcon = getSvgIcon('AI', { fontSize: '44px' })
273249
274-
const resolveMessageContent = (message: any) => {
275-
if (Array.isArray(message.renderContent) && message.renderContent.length > 0) {
276-
return resolveAgentRenderContent(message)
250+
const resolveMessageContent = (message: RobotMessage) => {
251+
if (props.messageContentResolver) {
252+
return props.messageContentResolver(message, {
253+
messages: messages.value,
254+
status: props.status
255+
})
277256
}
278257
279-
if (isAgentMessage(message) && message.role === 'assistant' && message.content) {
280-
const agentStatus = ['success', 'failed', 'fix'].includes(message.metadata?.agentStatus)
281-
? message.metadata.agentStatus
282-
: 'failed'
283-
return [
284-
{
285-
type: 'agent-content',
286-
status: agentStatus,
287-
content: message.content
258+
if (Array.isArray(message.renderContent) && message.renderContent.length > 0) {
259+
return message.renderContent.map((item) => {
260+
if (item?.type === RobotMessageContentType.Img || item?.type === RobotMessageContentType.Image) {
261+
return {
262+
type: RobotMessageContentType.Img,
263+
content: item.content || item.url || item.image_url?.url || ''
264+
}
288265
}
289-
]
266+
if (item?.type === RobotMessageContentType.Text) {
267+
return {
268+
type: RobotMessageContentType.Text,
269+
content: item.content ?? item.text ?? ''
270+
}
271+
}
272+
return item
273+
})
274+
}
275+
276+
if (Array.isArray(message.content) && message.content.length > 0) {
277+
const textContent = extractMessageText(message.content)
278+
if (textContent) {
279+
return textContent
280+
}
281+
}
282+
283+
const textContent = extractMessageText(message.content)
284+
if (textContent) {
285+
return textContent
290286
}
291287
292288
return message.content
293289
}
294290
295291
const roleConfigs: Record<string, BubbleRoleConfig> = {
296-
assistant: {
292+
[RobotMessageRole.Assistant]: {
297293
placement: 'start',
298294
avatar: aiAvatar
299295
},
300-
user: {
296+
[RobotMessageRole.User]: {
301297
placement: 'end'
302298
},
303-
system: {
299+
[RobotMessageRole.System]: {
304300
hidden: true
305301
}
306302
}
@@ -320,37 +316,38 @@ const handleSendMessage = async (content: string) => {
320316
return
321317
}
322318
323-
const userMessage: ChatMessage = {
324-
role: 'user',
319+
const userMessage: RobotMessage = {
320+
role: RobotMessageRole.User,
325321
content: messageContent
326322
}
327323
const files = selectedAttachments.value.filter((item) => item.status === 'success')
328324
if (files.length > 0) {
329-
const fileMessages: ChatMessage[] = files.map((file) => ({
330-
role: 'user',
331-
content: '',
332-
renderContent: [
333-
{
334-
type: 'img',
335-
content: file.url
336-
}
337-
]
338-
}))
339-
messages.value.push(...fileMessages)
340-
userMessage.content = files
341-
.map((item) => ({
342-
type: 'image_url',
325+
userMessage.content = [
326+
{
327+
type: RobotMessageContentType.Text,
328+
text: messageContent
329+
},
330+
...files.map((item) => ({
331+
type: RobotMessageContentType.ImageUrl,
343332
image_url: {
344333
url: item.url
345334
}
346335
}))
347-
.concat({
348-
type: 'text',
349-
text: messageContent
350-
})
336+
] as RobotInputContentPart[]
337+
userMessage.renderContent = [
338+
{
339+
type: RobotMessageContentType.Text,
340+
content: messageContent
341+
},
342+
...files.map((item) => ({
343+
type: RobotMessageContentType.Img,
344+
content: item.url
345+
}))
346+
]
347+
} else {
351348
userMessage.renderContent = [
352349
{
353-
type: 'text',
350+
type: RobotMessageContentType.Text,
354351
content: messageContent
355352
}
356353
]

0 commit comments

Comments
 (0)