diff --git a/README.md b/README.md index 50584cf..b485b5c 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,80 @@ Follow the [installation guide](https://docs.n8n.io/integrations/community-nodes - **Parameters:** - Message: `What is the latest update on project X?` - Agent: (Select from dropdown) - - Additional Fields (optional): Username, Email, Timezone + - Additional Fields (optional): Username, Email, Full Name, Timezone, Wait For Completion, Poll Interval (Ms), Max Wait (Ms), Include Generated File Content, Max Generated File Size (Bytes) **Output:** ``` { "agentMessage": "Project X is on track for delivery next week.", + "chainOfThought": "Looking up the project in Jira, then cross-referencing with the latest standup notes...", + "conversationId": "ivxVYkiwhA", + "conversationTitle": "Project X delivery update", "conversationUrl": "https://dust.tt/w/{workspaceId}/assistant/{conversationId}", + "generatedFiles": [ + { + "fileId": "fil_4aO6lmikIV63Bi", + "title": "jirateam_1780301864867_1.txt", + "contentType": "text/plain", + "snippet": "Found 70 issue(s)...", + "downloadUrl": "https://dust.tt/api/w/{workspaceId}/files/fil_4aO6lmikIV63Bi?action=download", + "fromToolName": "get_issues_using_jql", + "fromActionSId": "act_se8GjPQjCZzVwY", + "size": 161234, + "responseContentType": "text/plain; charset=utf-8", + "content": "Found 70 issue(s) using JQL\n\n# DATA-10998..." + } + ], + "rawConversation": { /* full conversation object including actions, citations, generated files */ }, "userMessage": { ... } } ``` +| Field | Description | +|---|---| +| `agentMessage` | Final answer text from the agent (markdown). | +| `chainOfThought` | Reasoning trace if the agent's model supports it (Sonnet/Opus thinking models). `null` otherwise. | +| `conversationId` | Dust `sId` of the conversation. Use it to send follow-up messages or to open the conversation in the Dust UI. | +| `conversationTitle` | Title auto-generated by Dust once the agent finishes. `null` if not set. | +| `conversationUrl` | Direct link to the conversation in the Dust UI. | +| `generatedFiles` | Array of files the agent generated during the run (e.g. CSV/text/HTML attachments produced by tool calls). Always populated with metadata; the actual content is inlined when **Include Generated File Content** is on. Empty array if the agent produced none. | +| `rawConversation` | The full conversation object (`content`, `actions`, `citations`, `generatedFiles`, …). Useful to extract tool outputs, file IDs, or per-action details downstream. Can be large for agents that make many tool calls. | +| `userMessage` | The first user message in the conversation, including the resolved user object. | + +#### Generated files + +When the agent produces files during its run (Jira extracts, CSV exports, HTML visualizations, …), they show up in the `generatedFiles` array. Each entry contains: + +| Sub-field | Description | +|---|---| +| `fileId` | Dust file ID (`fil_…`). | +| `title` | Display name of the file (e.g. `report.csv`). | +| `contentType` | Content-Type reported by Dust at generation time. | +| `snippet` | Short preview Dust attaches to the file metadata. | +| `downloadUrl` | Full HTTPS URL to `GET` the raw content (uses the same Dust API key for auth). | +| `fromToolName` / `fromActionSId` | Which tool call produced the file. `null` when the file is attached to the agent message itself. | +| `size` | Byte length of the downloaded content (only present when downloaded). | +| `responseContentType` | Content-Type returned by the download endpoint (only present when downloaded). | +| `content` | UTF-8 string content for textual files (HTML, CSV, JSON, Markdown, YAML, plain text, …). Only present when **Include Generated File Content** is on. | +| `contentBase64` | Base64-encoded content for binary files (images, PDFs, …). Only present when **Include Generated File Content** is on. | +| `tooLarge` | `true` if the file exceeded **Max Generated File Size (Bytes)** — content is omitted to keep the workflow output sane. | +| `downloadError` | Error message if the download itself failed. The rest of the workflow continues. | + +#### Long-running agents & polling + +The node creates a conversation and then **polls** the Dust API until the agent finishes, before returning the final `agentMessage`. This matches the pattern used by Dust's own clients and avoids `502 Bad Gateway` errors that previously occurred for agents that take more than ~30 seconds (e.g. agents that call out to Jira, Slack, Google Docs, or other slow tools). + +You can tune the polling behavior via Additional Fields: + +| Field | Default | What it does | +|---|---|---| +| `Wait For Completion` | `true` | If off, the node returns immediately with just `{ conversationId, conversationUrl, userMessage }`. Use this when you want to manage polling yourself (e.g. with an HTTP Request + Wait loop). | +| `Poll Interval (Ms)` | `1500` | Time between polling attempts while the agent is running. | +| `Max Wait (Ms)` | `120000` | Hard ceiling on the total wait. If the agent has not reached `succeeded` by then, the node throws. | + +If the agent ends in `failed` or `cancelled`, the node throws with the error message from Dust. The conversation URL is included in the error so you can inspect it in the Dust UI. + ### Upload a Document - **Operation:** Upload a Document @@ -103,6 +165,9 @@ Add these credentials in n8n under the "Dust API" credential type and reference ## Version history +- Unreleased + - Fix `502 Bad Gateway` for long-running agents by switching from blocking POST to non-blocking POST + polling on `GET /conversations/{id}`. Adds `Wait For Completion`, `Poll Interval (Ms)`, `Max Wait (Ms)`, and `Full Name` fields. Output now also includes `conversationId`, `chainOfThought`, `conversationTitle`, and `rawConversation`. + - Surface files the agent generates during the run as a top-level `generatedFiles` array. Adds `Include Generated File Content` (default on) and `Max Generated File Size (Bytes)` (default 10 MB) fields; textual files come back as inlined `content`, binary files as `contentBase64`. - 0.1.1 - Add SkipToolsValidation + rename assitant to agent - 0.1.0 diff --git a/nodes/Dust/Dust.node.ts b/nodes/Dust/Dust.node.ts index baaa0d3..bab3ac2 100644 --- a/nodes/Dust/Dust.node.ts +++ b/nodes/Dust/Dust.node.ts @@ -7,8 +7,34 @@ import { ILoadOptionsFunctions, INodePropertyOptions, NodeConnectionTypes, + NodeOperationError, } from 'n8n-workflow'; +declare const setTimeout: (handler: () => void, ms: number) => unknown; +declare const Buffer: { + from(data: unknown, encoding?: string): { length: number; toString(encoding?: string): string }; +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(() => resolve(), ms)); + +const TEXT_CONTENT_TYPE_HINTS = [ + 'text/', + 'json', + 'xml', + 'javascript', + 'html', + 'csv', + 'markdown', + 'yaml', + 'x-yaml', +]; + +const isTextualContentType = (contentType: string | null | undefined): boolean => { + if (!contentType) return false; + const ct = contentType.toLowerCase(); + return TEXT_CONTENT_TYPE_HINTS.some((hint) => ct.includes(hint)); +}; + export class Dust implements INodeType { description: INodeTypeDescription = { displayName: 'Dust', @@ -134,17 +160,50 @@ export class Dust implements INodeType { }, options: [ { - displayName: 'Username', - name: 'username', + displayName: 'Email', + name: 'email', type: 'string', + placeholder: 'name@email.com', default: '', }, { - displayName: 'Email', - name: 'email', + displayName: 'Full Name', + name: 'fullName', type: 'string', - placeholder: 'name@email.com', default: '', + description: 'Display name of the caller (e.g. "Jane Doe"). Used for attribution in Dust.', + }, + { + displayName: 'Include Generated File Content', + name: 'includeGeneratedFileContent', + type: 'boolean', + default: true, + description: + 'Whether to download the content of each file the agent generates (Jira extracts, CSVs, HTML visualizations, …) and include it inline in the output. Turn off to keep only metadata (fileId, title, downloadUrl).', + }, + { + displayName: 'Max Generated File Size (Bytes)', + name: 'maxGeneratedFileSizeBytes', + type: 'number', + default: 10000000, + description: + 'Per-file size ceiling when downloading generated file content. Larger files are reported with `tooLarge: true` and their content is omitted. Ignored when Include Generated File Content is off.', + }, + { + displayName: 'Max Wait (Ms)', + name: 'maxWaitMs', + type: 'number', + default: 120000, + description: + 'Maximum total time to wait for the agent to finish, in milliseconds. Ignored when Wait For Completion is off.', + }, + { + displayName: 'Poll Interval (Ms)', + name: 'pollIntervalMs', + type: 'number', + default: 1500, + description: + 'Time between polling attempts when waiting for the agent to finish, in milliseconds', }, { displayName: 'Timezone', @@ -152,6 +211,21 @@ export class Dust implements INodeType { type: 'string', default: '', }, + { + displayName: 'Username', + name: 'username', + type: 'string', + default: '', + description: 'Short identifier of the caller (e.g. "jane.doe"). Should not include the @domain part of an email.', + }, + { + displayName: 'Wait For Completion', + name: 'waitForCompletion', + type: 'boolean', + default: true, + description: + 'Whether to poll the conversation until the agent finishes and return its message. Turn off to return immediately with just the conversation ID.', + }, ], }, // Upload Document Parameters @@ -323,9 +397,28 @@ export class Dust implements INodeType { const baseUrl = credentials.region === 'EU' ? 'https://eu.dust.tt' : 'https://dust.tt'; const fullUrl = `${baseUrl}/api/v1/w/${credentials.workspaceId}/assistant/conversations`; + const waitForCompletion = + additionalFields.waitForCompletion === undefined + ? true + : (additionalFields.waitForCompletion as boolean); + const pollIntervalMs = Math.max( + 250, + (additionalFields.pollIntervalMs as number) || 1500, + ); + const maxWaitMs = Math.max( + pollIntervalMs, + (additionalFields.maxWaitMs as number) || 120000, + ); + const includeGeneratedFileContent = + additionalFields.includeGeneratedFileContent === undefined + ? true + : (additionalFields.includeGeneratedFileContent as boolean); + const maxGeneratedFileSizeBytes = Math.max( + 1, + (additionalFields.maxGeneratedFileSizeBytes as number) || 10000000, + ); + const body = { - blocking: true, - skipToolsValidation: true, title: null, visibility: 'unlisted', message: { @@ -334,7 +427,7 @@ export class Dust implements INodeType { timezone: additionalFields.timezone || 'Europe/Paris', username: additionalFields.username || 'DustN8N', email: additionalFields.email || 'n8n@dust.tt', - fullName: null, + fullName: (additionalFields.fullName as string) || null, profilePictureUrl: null, origin: 'n8n', }, @@ -356,31 +449,161 @@ export class Dust implements INodeType { }, }; - const response = await this.helpers.httpRequestWithAuthentication.call( + const createResponse = await this.helpers.httpRequestWithAuthentication.call( this, 'dustApi', requestOptions, ); - const conversationUrl = `${baseUrl}/w/${response.conversation.owner.sId}/assistant/${response.conversation.sId}`; + const conversationId = createResponse.conversation.sId; + const ownerSId = createResponse.conversation.owner.sId; + const conversationUrl = `${baseUrl}/w/${ownerSId}/assistant/${conversationId}`; - const agentMessages = response.conversation.content - .flat() - .filter((m: any) => m.type === 'agent_message') - .map((am: any) => am.content); + if (!waitForCompletion) { + const userMessage = createResponse.conversation.content.flat()[0]; + returnData.push({ + json: { + conversationId, + conversationUrl, + userMessage, + }, + pairedItem: { item: i }, + }); + continue; + } - const agentMessageStr = - agentMessages.length === 0 ? 'No message returned' : agentMessages.join('\n'); - const userMessage = response.conversation.content.flat()[0]; + const pollUrl = `${baseUrl}/api/v1/w/${credentials.workspaceId}/assistant/conversations/${conversationId}`; + const deadline = Date.now() + maxWaitMs; + let lastAgent: any = null; - returnData.push({ - json: { - agentMessage: agentMessageStr, - conversationUrl, - userMessage, - }, - pairedItem: { item: i }, - }); + while (Date.now() < deadline) { + await sleep(pollIntervalMs); + const poll = await this.helpers.httpRequestWithAuthentication.call(this, 'dustApi', { + method: 'GET' as IHttpRequestMethods, + url: pollUrl, + headers: { Accept: 'application/json' }, + }); + + const groups = (poll.conversation.content as any[][]) || []; + lastAgent = [...groups] + .reverse() + .flat() + .find((m: any) => m && m.type === 'agent_message'); + + if (lastAgent?.status === 'succeeded') { + const userMessage = poll.conversation.content.flat()[0]; + + const seenFileIds = new Set(); + const generatedFiles: IDataObject[] = []; + + const collectFile = ( + file: any, + source: { actionSId?: string; toolName?: string }, + ) => { + if (!file?.fileId || seenFileIds.has(file.fileId)) return; + seenFileIds.add(file.fileId); + generatedFiles.push({ + fileId: file.fileId, + title: file.title ?? null, + contentType: file.contentType ?? null, + snippet: file.snippet ?? null, + downloadUrl: `${baseUrl}/api/w/${credentials.workspaceId}/files/${file.fileId}?action=download`, + fromToolName: source.toolName ?? null, + fromActionSId: source.actionSId ?? null, + }); + }; + + if (Array.isArray(lastAgent.generatedFiles)) { + for (const file of lastAgent.generatedFiles) collectFile(file, {}); + } + if (Array.isArray(lastAgent.actions)) { + for (const action of lastAgent.actions) { + if (Array.isArray(action.generatedFiles)) { + for (const file of action.generatedFiles) { + collectFile(file, { + actionSId: action.sId, + toolName: action.toolName, + }); + } + } + } + } + + if (includeGeneratedFileContent && generatedFiles.length > 0) { + for (const file of generatedFiles) { + try { + const fileRes: any = + await this.helpers.httpRequestWithAuthentication.call( + this, + 'dustApi', + { + method: 'GET' as IHttpRequestMethods, + url: file.downloadUrl as string, + encoding: 'arraybuffer', + returnFullResponse: true, + headers: { Accept: '*/*' }, + }, + ); + + const responseContentType = + (fileRes.headers?.['content-type'] as string | undefined) ?? + (file.contentType as string | undefined) ?? + null; + const body = fileRes.body; + const size: number = body?.length ?? 0; + + file.size = size; + file.responseContentType = responseContentType; + + if (size > maxGeneratedFileSizeBytes) { + file.tooLarge = true; + continue; + } + + const buf = Buffer.from(body); + if (isTextualContentType(responseContentType)) { + file.content = buf.toString('utf-8'); + } else { + file.contentBase64 = buf.toString('base64'); + } + } catch (error) { + file.downloadError = (error as Error).message ?? 'unknown error'; + } + } + } + + returnData.push({ + json: { + agentMessage: lastAgent.content ?? '', + chainOfThought: lastAgent.chainOfThought ?? null, + conversationId, + conversationTitle: poll.conversation.title ?? null, + conversationUrl, + generatedFiles, + rawConversation: poll.conversation, + userMessage, + }, + pairedItem: { item: i }, + }); + break; + } + + if (lastAgent?.status === 'failed' || lastAgent?.status === 'cancelled') { + throw new NodeOperationError( + this.getNode(), + `Dust agent ${lastAgent.status}: ${lastAgent.error?.message ?? 'unknown error'}`, + { itemIndex: i }, + ); + } + } + + if (lastAgent?.status !== 'succeeded') { + throw new NodeOperationError( + this.getNode(), + `Dust agent did not complete within ${maxWaitMs}ms (last status: ${lastAgent?.status ?? 'unknown'}). Conversation: ${conversationUrl}`, + { itemIndex: i }, + ); + } } else if (operation === 'uploadDocument') { const spaceId = this.getNodeParameter('spaceId', i) as string; const dataSourceName = this.getNodeParameter('dataSourceName', i) as string;