Skip to content

Commit 81d0d8f

Browse files
andris9andris9
authored
feat: Attachments in new message webhooks (#514)
* Added configuration settings to allow including attachment content in webhooks * Include attachments in new email webhooks if configured
1 parent a8acde2 commit 81d0d8f

File tree

8 files changed

+181
-8
lines changed

8 files changed

+181
-8
lines changed

‎lib/email-client/base-client.js

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1935,6 +1935,24 @@ class BaseClient {
19351935
}
19361936
}
19371937

1938+
let notifyAttachments = await settings.get('notifyAttachments');
1939+
let notifyAttachmentSize = await settings.get('notifyAttachmentSize');
1940+
if (notifyAttachments && messageData.attachments?.length) {
1941+
for (let attachment of messageData.attachments || []) {
1942+
if (notifyAttachmentSize && attachment.encodedSize && attachment.encodedSize > notifyAttachmentSize) {
1943+
// skip large attachments
1944+
continue;
1945+
}
1946+
if (!attachment.content) {
1947+
try {
1948+
attachment.content = (await this.getAttachment(attachment.id))?.data?.toString('base64');
1949+
} catch (err) {
1950+
this.logger.error({ msg: 'Failed to load attachment content', attachment, err });
1951+
}
1952+
}
1953+
}
1954+
}
1955+
19381956
if (messageData.attachments && messageData.attachments.length && messageData.text && messageData.text.html) {
19391957
// fetch inline attachments
19401958
for (let attachment of messageData.attachments) {

‎lib/email-client/imap/mailbox.js

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -866,7 +866,36 @@ class Mailbox {
866866
}
867867
}
868868

869-
if (messageInfo.attachments && messageInfo.attachments.length && messageInfo.text && messageInfo.text.html) {
869+
let notifyAttachments = await settings.get('notifyAttachments');
870+
let notifyAttachmentSize = await settings.get('notifyAttachmentSize');
871+
if (notifyAttachments && messageInfo.attachments?.length) {
872+
for (let attachment of messageInfo.attachments || []) {
873+
if (notifyAttachmentSize && attachment.encodedSize && attachment.encodedSize > notifyAttachmentSize) {
874+
// skip large attachments
875+
continue;
876+
}
877+
if (!attachment.content) {
878+
try {
879+
let buf = Buffer.from(attachment.id, 'base64url');
880+
let part = buf.subarray(8).toString();
881+
882+
let { content: downloadStream } = await this.connection.imapClient.download(messageInfo.uid, part, {
883+
uid: true,
884+
chunkSize: options.chunkSize,
885+
maxBytes: MAX_ALLOWED_DOWNLOAD_SIZE
886+
});
887+
888+
if (downloadStream) {
889+
attachment.content = (await download(downloadStream)).toString('base64');
890+
}
891+
} catch (err) {
892+
this.logger.error({ msg: 'Failed to load attachment content', attachment, err });
893+
}
894+
}
895+
}
896+
}
897+
898+
if (messageInfo.attachments?.length && messageInfo.text?.html) {
870899
// fetch inline attachments
871900
for (let attachment of messageInfo.attachments) {
872901
if (attachment.encodedSize && attachment.encodedSize > MAX_INLINE_ATTACHMENT_SIZE) {

‎lib/email-client/outlook-client.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2522,7 +2522,7 @@ class OutlookClient extends BaseClient {
25222522
// The message still exists, so there's no need to notify about the deletion.
25232523
// The message was likely moved, but since we cannot determine the previous mailbox folder,
25242524
// there is no point in notifying about it.
2525-
this.logger.debug({ msg: 'Ignore deleted email event. Still exists.', type: 'history-event', emailId });
2525+
this.logger.debug({ msg: 'Ignore deleted email event. Still exists.', type: 'history-event', emailId, messageData });
25262526
return;
25272527
}
25282528

‎lib/routes-ui.js

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const {
2020
readEnvValue,
2121
getServiceHostname,
2222
getByteSize,
23+
formatByteSize,
2324
getDuration,
2425
parseSignedFormData,
2526
updatePublicInterfaces,
@@ -275,10 +276,35 @@ const configWebhooksSchema = {
275276
notifyHeaders: Joi.string().empty('').trim(),
276277
notifyText: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false),
277278
notifyWebSafeHtml: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false),
278-
notifyTextSize: Joi.number().integer().empty(''),
279+
280+
notifyTextSize: Joi.alternatives().try(
281+
Joi.number().empty('').integer().min(0),
282+
// If it's a string, parse and convert it to bytes
283+
Joi.string().custom((value, helpers) => {
284+
let nr = getByteSize(value);
285+
if (typeof nr !== 'number' || nr < 0) {
286+
return helpers.error('any.invalid');
287+
}
288+
return nr;
289+
}, 'Byte size conversion')
290+
),
291+
279292
notifyCalendarEvents: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false),
280293
inboxNewOnly: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false),
281294

295+
notifyAttachments: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').default(false),
296+
notifyAttachmentSize: Joi.alternatives().try(
297+
Joi.number().empty('').integer().min(0),
298+
// If it's a string, parse and convert it to bytes
299+
Joi.string().custom((value, helpers) => {
300+
let nr = getByteSize(value);
301+
if (typeof nr !== 'number' || nr < 0) {
302+
return helpers.error('any.invalid');
303+
}
304+
return nr;
305+
}, 'Byte size conversion')
306+
),
307+
282308
customHeaders: Joi.string()
283309
.allow('')
284310
.trim()
@@ -1029,6 +1055,10 @@ function applyRoutes(server, call) {
10291055
const notifyWebSafeHtml = (await settings.get('notifyWebSafeHtml')) || false;
10301056
const notifyTextSize = Number(await settings.get('notifyTextSize')) || 0;
10311057
const notifyCalendarEvents = (await settings.get('notifyCalendarEvents')) || false;
1058+
1059+
const notifyAttachments = (await settings.get('notifyAttachments')) || false;
1060+
const notifyAttachmentSize = Number(await settings.get('notifyAttachmentSize')) || 0;
1061+
10321062
const inboxNewOnly = (await settings.get('inboxNewOnly')) || false;
10331063
const customHeaders = (await settings.get('webhooksCustomHeaders')) || [];
10341064

@@ -1047,9 +1077,12 @@ function applyRoutes(server, call) {
10471077
.join('\n'),
10481078
notifyText,
10491079
notifyWebSafeHtml,
1050-
notifyTextSize: notifyTextSize ? notifyTextSize : '',
1080+
notifyTextSize: notifyTextSize ? formatByteSize(notifyTextSize) : '',
10511081
notifyCalendarEvents,
10521082

1083+
notifyAttachments,
1084+
notifyAttachmentSize: notifyAttachmentSize ? formatByteSize(notifyAttachmentSize) : '',
1085+
10531086
customHeaders: []
10541087
.concat(customHeaders || [])
10551088
.map(entry => `${entry.key}: ${entry.value}`.trim())
@@ -1109,6 +1142,10 @@ function applyRoutes(server, call) {
11091142
notifyWebSafeHtml: request.payload.notifyWebSafeHtml,
11101143
notifyTextSize: request.payload.notifyTextSize || 0,
11111144
notifyCalendarEvents: request.payload.notifyCalendarEvents,
1145+
1146+
notifyAttachments: request.payload.notifyAttachments,
1147+
notifyAttachmentSize: request.payload.notifyAttachmentSize,
1148+
11121149
inboxNewOnly: request.payload.inboxNewOnly,
11131150

11141151
webhookEvents: notificationTypes.filter(type => !!request.payload[`notify_${type.name}`]).map(type => type.name),

‎lib/schemas.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,9 @@ const settingsSchema = {
205205

206206
notifyTextSize: Joi.number().integer().min(0).description('Maximum size of message text to include in webhook notifications (in bytes)'),
207207

208+
notifyAttachments: Joi.boolean().truthy('Y', 'true', '1', 'on').falsy('N', 'false', 0, '').description('Include attachments in webhook notifications'),
209+
notifyAttachmentSize: Joi.number().integer().min(0).description('Maximum size of attachment to include in webhook notifications (in bytes)'),
210+
208211
notifyCalendarEvents: Joi.boolean()
209212
.truthy('Y', 'true', '1', 'on')
210213
.falsy('N', 'false', 0, '')

‎lib/tools.js

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,29 @@ module.exports = {
496496
return Number(val);
497497
},
498498

499+
formatByteSize(val) {
500+
if (isNaN(val)) {
501+
return val;
502+
}
503+
val = Number(val);
504+
505+
let types = new Set([
506+
['PB', 1024 * 1024 * 1024 * 1024 * 1024],
507+
['TB', 1024 * 1024 * 1024 * 1024],
508+
['GB', 1024 * 1024 * 1024],
509+
['MB', 1024 * 1024],
510+
['kB', 1024]
511+
]);
512+
513+
for (let [[type, nr]] of types.entries()) {
514+
if (val % nr === 0) {
515+
return `${Math.round(val / nr)}${type}`;
516+
}
517+
}
518+
519+
return val;
520+
},
521+
499522
formatAccountListingResponse(entry) {
500523
if (Array.isArray(entry)) {
501524
let obj = {};
@@ -1834,14 +1857,14 @@ MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEV3QUiYsp13nD9suD1/ZkEXnuMoSg
18341857
while (redirect++ < hopsAllowed) {
18351858
let oauth2DataStr = await redis.hget(`${REDIS_PREFIX}iad:${account}`, 'oauth2');
18361859
if (!oauth2DataStr) {
1837-
let error = new Error(`Missing account data for delegated account "account"`);
1860+
let error = new Error(`Missing account data for delegated account "${account}"`);
18381861
throw error;
18391862
}
18401863
let oauth2Data;
18411864
try {
18421865
oauth2Data = JSON.parse(oauth2DataStr);
18431866
} catch (err) {
1844-
let error = new Error(`Invalid account data for delegated account "account"`);
1867+
let error = new Error(`Invalid account data for delegated account "${account}"`);
18451868
throw error;
18461869
}
18471870

‎lib/webhooks.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,8 +488,15 @@ class WebhooksHandler {
488488

489489
// remove attachment contents
490490
if (event === MESSAGE_NEW_NOTIFY && payload && payload.data && payload.data.attachments) {
491+
let notifyAttachments = await settings.get('notifyAttachments');
492+
let notifyAttachmentSize = await settings.get('notifyAttachmentSize');
493+
491494
for (let attachment of payload.data.attachments) {
492495
if (attachment.content) {
496+
if (notifyAttachments && (!notifyAttachmentSize || notifyAttachmentSize > (attachment.content.length / 4) * 3)) {
497+
// keep the attachment
498+
continue;
499+
}
493500
delete attachment.content;
494501
}
495502
}

‎views/config/webhooks.hbs

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -284,8 +284,9 @@
284284
target="_blank" rel="noopener noreferrer">notifyTextSize</a>]
285285
</div>
286286
<label for="settingsNotifyTextSize">Maximum Text Size</label>
287-
<input type="number" class="form-control text-list {{#if errors.notifyTextSize}}is-invalid{{/if}}"
288-
id="settingsNotifyTextSize" min="0" name="notifyTextSize" value="{{values.notifyTextSize}}" />
287+
<input type="text" class="form-control text-list {{#if errors.notifyTextSize}}is-invalid{{/if}}"
288+
id="settingsNotifyTextSize" min="0" name="notifyTextSize" value="{{values.notifyTextSize}}"
289+
placeholder="For example: &quot;1MB&quot;" />
289290
{{#if errors.notifyTextSize}}
290291
<span class="invalid-feedback">{{errors.notifyTextSize}}</span>
291292
{{/if}}
@@ -315,6 +316,61 @@
315316
</div>
316317
</div>
317318

319+
<div class="card mt-4 mb-4">
320+
<div class="card-header py-3">
321+
<h6 class="m-0 font-weight-bold text-primary">
322+
Attachments
323+
<small class="text-muted">
324+
&mdash; Applies to all webhooks, including
325+
<a href="/admin/webhooks">routed webhooks</a>
326+
</small>
327+
</h6>
328+
</div>
329+
<div class="card-body">
330+
<p>By default, attachments are not included in new email webhook payloads.</p>
331+
332+
<div class="form-group form-check">
333+
<div class="text-muted float-right code-link">
334+
[<a href="/admin/swagger#/Settings/postV1Settings" target="_blank"
335+
rel="noopener noreferrer">notifyAttachments</a>]
336+
</div>
337+
<input type="checkbox"
338+
class="form-check-input or-else-all {{#if errors.notifyAttachments}}is-invalid{{/if}}"
339+
data-target="attachment-list" data-reverse="true" id="settingsNotifyAttachments"
340+
name="notifyAttachments" {{#if values.notifyAttachments}}checked{{/if}} />
341+
<label class="form-check-label" for="settingsNotifyAttachments">
342+
Include attachments in new email webhooks. The attachment content is base64-encoded.
343+
</label>
344+
{{#if errors.notifyAttachments}}
345+
<span class="invalid-feedback">{{errors.notifyAttachments}}</span>
346+
{{/if}}
347+
</div>
348+
349+
<div class="form-group">
350+
<div class="text-muted float-right code-link">
351+
[<a href="/admin/swagger#/Settings/postV1Settings" target="_blank"
352+
rel="noopener noreferrer">notifyAttachmentSize</a>]
353+
</div>
354+
<label for="settingsNotifyAttachmentSize">Maximum Attachment Size</label>
355+
<input type="text"
356+
class="form-control attachment-list {{#if errors.notifyAttachmentSize}}is-invalid{{/if}}"
357+
id="settingsNotifyAttachmentSize" min="0" name="notifyAttachmentSize"
358+
value="{{values.notifyAttachmentSize}}" placeholder="Example: &quot;2MB&quot;" />
359+
{{#if errors.notifyAttachmentSize}}
360+
<span class="invalid-feedback">{{errors.notifyAttachmentSize}}</span>
361+
{{/if}}
362+
<small class="form-text text-muted">
363+
Specify the maximum file size for attachments to keep the webhook payload manageable.
364+
Attachments larger than this limit are skipped and not included in the webhook payload.
365+
<br>
366+
<strong>Warning:</strong> EmailEngine stores unposted webhook payloads in Redis. If many
367+
large attachments are processed, these cached payloads can quickly fill up the Redis DB.
368+
</small>
369+
</div>
370+
</div>
371+
</div>
372+
373+
318374
<div class="mb-4">
319375
<button type="submit" class="btn btn-primary btn-icon-split">
320376
<span class="icon text-white-50">

0 commit comments

Comments
 (0)