Skip to content

Commit e811cfd

Browse files
committed
feat(assistant): add AI diagram disclosure
Persist disclosure labels and AI provenance metadata for assistant-generated Mermaid diagrams. Keep generated assistant diagrams on the whiteboard default font path so labels do not render with the Excalidraw fallback font or clip from mismatched metrics. Signed-off-by: Hoang Pham <hoangmaths96@gmail.com>
1 parent faca443 commit e811cfd

4 files changed

Lines changed: 302 additions & 28 deletions

File tree

playwright/e2e/assistant.spec.ts

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,32 @@
44
*/
55
import { test } from '../support/fixtures/random-user'
66
import { expect } from '@playwright/test'
7-
import { createWhiteboard, openFilesApp } from '../support/utils'
7+
import type { Page } from '@playwright/test'
8+
import {
9+
captureBoardAuthFromSave,
10+
createWhiteboard,
11+
fetchBoardContent,
12+
openFilesApp,
13+
openWhiteboardFromFiles,
14+
} from '../support/utils'
15+
16+
const MODAL_DISCLOSURE_TEXT = 'Output is generated by AI. Make sure to always double-check.'
17+
const DIAGRAM_DISCLOSURE_TEXT = 'Diagram shown here is generated by AI. Make sure to always double-check.'
18+
const AI_GENERATED_TYPE = 'assistant-mermaid'
19+
const normalizeText = (text = '') => text.replace(/\s+/g, ' ').trim()
20+
21+
const ensureAssistantBoardReady = async (page: Page, boardName: string) => {
22+
const assistantButton = page.getByRole('button', { name: 'Assistant', exact: true })
23+
24+
try {
25+
await expect(assistantButton).toBeVisible({ timeout: 15000 })
26+
} catch {
27+
await openWhiteboardFromFiles(page, boardName)
28+
await expect(assistantButton).toBeVisible({ timeout: 30000 })
29+
}
30+
31+
return assistantButton
32+
}
833

934
test.beforeEach(async ({ page }) => {
1035
let taskIdCounter = 1
@@ -92,10 +117,32 @@ test.beforeEach(async ({ page }) => {
92117
test('Assistant Button', async ({ page }) => {
93118
const boardName = `Assistant ${Date.now()}`
94119
await createWhiteboard(page, { name: boardName })
95-
await page.getByRole('button', { name: 'Assistant', exact: true }).click()
120+
const assistantButton = await ensureAssistantBoardReady(page, boardName)
121+
await assistantButton.click()
122+
await expect(page.getByText(MODAL_DISCLOSURE_TEXT)).toBeVisible()
96123
await page.getByRole('textbox', { name: 'Prompt to generate diagram' }).fill('abc')
124+
const authFromSave = captureBoardAuthFromSave(page)
97125
await page.getByRole('button', { name: 'Generate' }).click()
98126
await expect(page.getByRole('textbox', { name: 'Prompt to generate diagram' })).not.toBeVisible()
127+
const { fileId, jwt } = await authFromSave
128+
const auth = { fileId, jwt }
129+
130+
await expect.poll(async () => {
131+
const content = await fetchBoardContent(page, auth)
132+
return content.elements.some((element) => normalizeText(element.text) === DIAGRAM_DISCLOSURE_TEXT)
133+
}, {
134+
timeout: 30000,
135+
interval: 500,
136+
}).toBe(true)
137+
138+
const content = await fetchBoardContent(page, auth)
139+
const disclosureElement = content.elements.find((element) => normalizeText(element.text) === DIAGRAM_DISCLOSURE_TEXT)
140+
141+
expect(disclosureElement).toBeTruthy()
142+
expect(disclosureElement.customData?.aiDisclosureLabel).toBe(true)
143+
expect(disclosureElement.customData?.aiGenerated).toBe(AI_GENERATED_TYPE)
144+
expect(content.elements.length).toBeGreaterThan(0)
145+
expect(content.elements.every((element) => element.customData?.aiGenerated === AI_GENERATED_TYPE)).toBe(true)
99146
})
100147

101148
test('Restart on false Assistant output', async ({ page }) => {
@@ -144,8 +191,11 @@ test('Restart on false Assistant output', async ({ page }) => {
144191
)
145192
const boardName = `Assistant invalid ${Date.now()}`
146193
await createWhiteboard(page, { name: boardName })
147-
await page.getByRole('button', { name: 'Assistant', exact: true }).click()
194+
const assistantButton = await ensureAssistantBoardReady(page, boardName)
195+
await assistantButton.click()
196+
await expect(page.getByText(MODAL_DISCLOSURE_TEXT)).toBeVisible()
148197
await page.getByRole('textbox', { name: 'Prompt to generate diagram' }).fill('abc')
149198
await page.getByRole('button', { name: 'Generate' }).click()
150199
await expect(page.getByRole('textbox', { name: 'Prompt to generate diagram' })).toHaveValue('abc')
200+
await expect(page.getByText(MODAL_DISCLOSURE_TEXT)).toBeVisible()
151201
})

src/components/AssistantDialog.vue

Lines changed: 220 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,20 @@ import { NcTextField, NcButton, NcModal } from '@nextcloud/vue'
77
import { t } from '@nextcloud/l10n'
88
import { ScheduleTask } from '../services/assistant/api'
99
import { parseMermaidToExcalidraw } from '@excalidraw/mermaid-to-excalidraw'
10-
import { convertToExcalidrawElements } from '@nextcloud/excalidraw'
10+
import { convertToExcalidrawElements, getCommonBounds } from '@nextcloud/excalidraw'
1111
import { defineComponent } from 'vue'
1212
13+
const AI_GENERATED_TYPE = 'assistant-mermaid'
14+
const WHITEBOARD_DEFAULT_FONT_FAMILY = 3
15+
const WHITEBOARD_DEFAULT_FONT = 'Cascadia, monospace'
16+
const AI_DISCLOSURE_LABEL_GAP = 16
17+
const AI_DISCLOSURE_LABEL_FONT_FAMILY = 2
18+
const AI_DISCLOSURE_LABEL_FONT = 'Helvetica, Arial, sans-serif'
19+
const AI_DISCLOSURE_LABEL_FONT_SIZE = 14
20+
const AI_DISCLOSURE_LABEL_MIN_WIDTH = 280
21+
const AI_DISCLOSURE_LABEL_MAX_WIDTH = 440
22+
const AI_DISCLOSURE_LABEL_FONT_STRING = `${AI_DISCLOSURE_LABEL_FONT_SIZE}px ${AI_DISCLOSURE_LABEL_FONT}`
23+
1324
export default defineComponent({
1425
name: 'AssistantDialog',
1526
components: {
@@ -40,29 +51,173 @@ export default defineComponent({
4051
},
4152
methods: {
4253
t,
54+
createAssistantMetadata(extraData = {}) {
55+
return {
56+
aiGenerated: AI_GENERATED_TYPE,
57+
...extraData,
58+
}
59+
},
60+
getAssistantDisclosureText() {
61+
return t('whiteboard', 'Output is generated by AI. Make sure to always double-check.')
62+
},
63+
getDiagramDisclosureText() {
64+
return t('whiteboard', 'Diagram shown here is generated by AI. Make sure to always double-check.')
65+
},
4366
onCancel() {
4467
this.show = false
4568
this.$emit('cancel')
4669
},
47-
async getExcalidrawElements(taskResponse) {
48-
this.mermaidError = false
49-
50-
const { elements, files } = await parseMermaidToExcalidraw(taskResponse)
70+
markAssistantGeneratedElements(elements) {
71+
return elements.map((element) => ({
72+
...element,
73+
customData: {
74+
...element.customData,
75+
...this.createAssistantMetadata(),
76+
},
77+
}))
78+
},
79+
normalizeAssistantElements(elements) {
5180
elements.forEach((element) => {
52-
// set font family (6 should always be Nunito)
5381
if (element.label) {
54-
element.label.fontFamily = 6
82+
element.label.fontFamily = WHITEBOARD_DEFAULT_FONT_FAMILY
5583
}
5684
if (element.type === 'text') {
57-
element.fontFamily = 6
85+
element.fontFamily = WHITEBOARD_DEFAULT_FONT_FAMILY
5886
}
5987
element.roughness = 0
6088
})
61-
const data = {
62-
elements: convertToExcalidrawElements(elements, { regenerateIds: true }),
89+
},
90+
measureDisclosureTextWidth(text) {
91+
if (typeof document === 'undefined') {
92+
return text.length * AI_DISCLOSURE_LABEL_FONT_SIZE * 0.55
93+
}
94+
95+
const canvas = document.createElement('canvas')
96+
const context = canvas.getContext('2d')
97+
98+
if (!context) {
99+
return text.length * AI_DISCLOSURE_LABEL_FONT_SIZE * 0.55
100+
}
101+
102+
context.font = AI_DISCLOSURE_LABEL_FONT_STRING
103+
return context.measureText(text).width
104+
},
105+
splitDisclosureWord(word, maxWidth) {
106+
const segments = []
107+
let currentSegment = ''
108+
109+
for (const character of word) {
110+
const nextSegment = `${currentSegment}${character}`
111+
112+
if (currentSegment && this.measureDisclosureTextWidth(nextSegment) > maxWidth) {
113+
segments.push(currentSegment)
114+
currentSegment = character
115+
continue
116+
}
117+
118+
currentSegment = nextSegment
119+
}
120+
121+
if (currentSegment) {
122+
segments.push(currentSegment)
123+
}
124+
125+
return segments
126+
},
127+
wrapDisclosureText(text, maxWidth) {
128+
const words = text.split(/\s+/).filter(Boolean)
129+
130+
if (!words.length) {
131+
return text
132+
}
133+
134+
const lines = []
135+
let currentLine = ''
136+
137+
words.forEach((word) => {
138+
const segments = this.measureDisclosureTextWidth(word) > maxWidth
139+
? this.splitDisclosureWord(word, maxWidth)
140+
: [word]
141+
142+
segments.forEach((segment, index) => {
143+
const nextLine = currentLine ? `${currentLine} ${segment}` : segment
144+
145+
if (this.measureDisclosureTextWidth(nextLine) <= maxWidth) {
146+
currentLine = nextLine
147+
} else {
148+
if (currentLine) {
149+
lines.push(currentLine)
150+
}
151+
currentLine = segment
152+
}
153+
154+
if (index < segments.length - 1 && currentLine) {
155+
lines.push(currentLine)
156+
currentLine = ''
157+
}
158+
})
159+
})
160+
161+
if (currentLine) {
162+
lines.push(currentLine)
163+
}
164+
165+
return lines.join('\n')
166+
},
167+
addAssistantDisclosureLabel(elements) {
168+
if (!elements.length) {
169+
return elements
170+
}
171+
172+
const [minX, , maxX, maxY] = getCommonBounds(elements)
173+
const diagramWidth = maxX - minX
174+
const disclosureText = this.wrapDisclosureText(
175+
this.getDiagramDisclosureText(),
176+
Math.min(
177+
AI_DISCLOSURE_LABEL_MAX_WIDTH,
178+
Math.max(AI_DISCLOSURE_LABEL_MIN_WIDTH, diagramWidth * 0.8),
179+
),
180+
)
181+
const [disclosureElement] = convertToExcalidrawElements([{
182+
type: 'text',
183+
x: minX,
184+
y: maxY + AI_DISCLOSURE_LABEL_GAP,
185+
text: disclosureText,
186+
fontSize: AI_DISCLOSURE_LABEL_FONT_SIZE,
187+
fontFamily: AI_DISCLOSURE_LABEL_FONT_FAMILY,
188+
textAlign: 'center',
189+
backgroundColor: 'transparent',
190+
strokeColor: '#6b7280',
191+
opacity: 80,
192+
customData: this.createAssistantMetadata({
193+
aiDisclosureLabel: true,
194+
}),
195+
}], { regenerateIds: true })
196+
197+
return [
198+
...elements,
199+
{
200+
...disclosureElement,
201+
x: minX + ((diagramWidth - disclosureElement.width) / 2),
202+
y: maxY + AI_DISCLOSURE_LABEL_GAP,
203+
},
204+
]
205+
},
206+
async getExcalidrawElements(taskResponse) {
207+
this.mermaidError = false
208+
209+
const { elements, files } = await parseMermaidToExcalidraw(taskResponse, {
210+
fontFamily: WHITEBOARD_DEFAULT_FONT,
211+
})
212+
this.normalizeAssistantElements(elements)
213+
214+
const generatedElements = this.markAssistantGeneratedElements(
215+
convertToExcalidrawElements(elements, { regenerateIds: true }),
216+
)
217+
return {
218+
elements: this.addAssistantDisclosureLabel(generatedElements),
63219
files,
64220
}
65-
return data
66221
},
67222
async onGetTask() {
68223
this.waitingForTask = true
@@ -96,6 +251,12 @@ export default defineComponent({
96251
<h2>
97252
{{ t('whiteboard', 'Generate diagram') }}
98253
</h2>
254+
<div class="assistant-disclosure">
255+
<span class="assistant-disclosure__badge">AI</span>
256+
<p class="assistant-disclosure__text">
257+
{{ getAssistantDisclosureText() }}
258+
</p>
259+
</div>
99260
<div v-if="mermaidError" class="mermaid-error">
100261
{{ t('whiteboard', 'Something went wrong, please try again') }}
101262
</div>
@@ -123,27 +284,65 @@ export default defineComponent({
123284
.loading-icon {
124285
padding: 30px;
125286
}
287+
126288
h2 {
127-
text-align: center;
289+
text-align: center;
128290
}
291+
129292
.assistant-dialog {
130-
display: flex;
131-
flex-direction: column;
132-
justify-content: space-evenly;
133-
overflow: hidden;
293+
display: flex;
294+
flex-direction: column;
295+
justify-content: space-evenly;
296+
overflow: hidden;
297+
134298
form {
135299
padding-inline: 10px;
136300
}
137301
}
302+
303+
.assistant-disclosure {
304+
display: flex;
305+
align-items: flex-start;
306+
gap: 10px;
307+
margin: 0 10px 14px;
308+
padding: 8px 10px;
309+
border: 1px solid transparent;
310+
border-radius: 6px;
311+
background:
312+
linear-gradient(var(--color-background-assistant), var(--color-background-assistant)) padding-box,
313+
var(--color-border-assistant) border-box;
314+
color: var(--color-main-text);
315+
font-size: var(--font-size-small);
316+
line-height: 1.35;
317+
}
318+
319+
.assistant-disclosure__badge {
320+
flex: none;
321+
min-width: 28px;
322+
padding: 2px 6px;
323+
border-radius: 6px;
324+
background: var(--color-border-assistant);
325+
font-size: 11px;
326+
font-weight: 700;
327+
line-height: 1.4;
328+
text-align: center;
329+
}
330+
331+
.assistant-disclosure__text {
332+
margin: 0;
333+
font-weight: 500;
334+
}
335+
138336
.dialog-buttons {
139-
display: flex;
140-
justify-content: space-between;
141-
align-items: center;
142-
width: 100%;
337+
display: flex;
338+
justify-content: space-between;
339+
align-items: center;
340+
width: 100%;
143341
padding-block: 10px;
144342
}
343+
145344
.mermaid-error {
146345
text-align: center;
147-
font-weight: bold;
346+
font-weight: bold;
148347
}
149348
</style>

src/types/whiteboard.d.ts

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,16 @@ export interface ElementCreatorInfo {
1010
createdAt: number
1111
}
1212

13+
export interface WhiteboardElementCustomData {
14+
creator?: ElementCreatorInfo
15+
lastModifiedBy?: ElementCreatorInfo
16+
aiGenerated?: 'assistant-mermaid'
17+
aiDisclosureLabel?: true
18+
[key: string]: unknown
19+
}
20+
1321
export type WhiteboardElement ={
14-
customData?: {
15-
creator?: ElementCreatorInfo
16-
lastModifiedBy?: ElementCreatorInfo
17-
},
22+
customData?: WhiteboardElementCustomData,
1823
} & ExcalidrawLinearElement & ExcalidrawElement
1924

2025
export interface CreatorDisplaySettings {

0 commit comments

Comments
 (0)