diff --git a/src/mastra/chat/commands/stop.ts b/src/mastra/chat/commands/stop.ts index 68bd506..bc2695c 100644 --- a/src/mastra/chat/commands/stop.ts +++ b/src/mastra/chat/commands/stop.ts @@ -3,6 +3,10 @@ import { memoryThread } from '../../lib/memory'; import type { CommandHandler } from '../../types'; export const stop: CommandHandler = async ({ message, thread }) => { + // Mark first: RFC i requires every later message in this thread to be + // ignored even if aborting the current run below fails. + await thread.setState({ stopped: true }); + const { default: orchestrator } = await import('../../agents/orchestrator'); const threadMemory = await memoryThread({ agent: orchestrator, @@ -37,22 +41,7 @@ export const stop: CommandHandler = async ({ message, thread }) => { } })(); - if (!(scope && (activeRunId || backgroundTasks.length > 0))) { - await thread - .postEphemeral(message.author, 'Nothing to stop right now.', { - fallbackToDM: false, - }) - .catch((error: unknown) => { - logger.warn('[commands] Failed to post stop feedback', { - error, - threadId: thread.id, - userId: message.author.userId, - }); - }); - return; - } - - if (activeRunId) { + if (scope && activeRunId) { orchestrator.abortThreadStream(scope); } if (manager) { diff --git a/src/mastra/chat/guardrails.test.ts b/src/mastra/chat/guardrails.test.ts new file mode 100644 index 0000000..af9caab --- /dev/null +++ b/src/mastra/chat/guardrails.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, test } from 'bun:test'; +import { blockedByAgentGuidelines } from './guardrails'; + +const BOT_USER_ID = 'UBOTTEST01'; + +const activeThread = { + state: Promise.resolve({ respondOnThreadMessages: true }), +}; +const stoppedThread = { state: Promise.resolve({ stopped: true }) }; + +function message(raw: string): { raw: unknown; text: string } { + return { raw: { text: raw }, text: raw }; +} + +function blocked({ + botUserId = BOT_USER_ID, + raw, + thread = activeThread, +}: { + botUserId?: string | undefined; + raw: string; + thread?: { readonly state: Promise }; +}) { + return blockedByAgentGuidelines({ + botUserId, + message: message(raw), + thread, + }); +} + +describe('rule 1: double-hash comments', () => { + test('ignores a message starting with ##', async () => { + expect(await blocked({ raw: '## hidden from the bot' })).toBe(true); + }); + + test('ignores ## even when the bot is directly mentioned', async () => { + expect(await blocked({ raw: `<@${BOT_USER_ID}> ## note` })).toBe(true); + }); + + test('leading whitespace still counts', async () => { + expect(await blocked({ raw: ' ## note' })).toBe(true); + }); + + test('a ## on any line counts', async () => { + expect(await blocked({ raw: 'hello\n## subheading' })).toBe(true); + }); + + test('a single # is not a comment', async () => { + expect(await blocked({ raw: '# regular heading' })).toBe(false); + }); + + test('### counts as a comment prefix', async () => { + expect(await blocked({ raw: '### deep heading' })).toBe(true); + }); +}); + +describe('rule 2: !stop persistence gate', () => { + test('a stopped thread ignores later messages', async () => { + expect(await blocked({ raw: 'hello again', thread: stoppedThread })).toBe( + true + ); + }); + + test('a stopped thread ignores direct mentions too', async () => { + expect( + await blocked({ + raw: `<@${BOT_USER_ID}> hello`, + thread: stoppedThread, + }) + ).toBe(true); + }); + + test('an active thread still processes messages', async () => { + expect(await blocked({ raw: 'hello again' })).toBe(false); + }); +}); + +describe('rule 3: ping group mentions', () => { + test('ignores a usergroup mention without a bot mention', async () => { + expect( + await blocked({ raw: ' please take a look' }) + ).toBe(true); + }); + + test('ignores legacy subteam tokens', async () => { + expect(await blocked({ raw: ' ping' })).toBe(true); + }); + + test('ignores labeled usergroup tokens', async () => { + expect(await blocked({ raw: ' ping' })).toBe( + true + ); + }); + + test('ignores @here, @channel, and @everyone broadcasts', async () => { + expect(await blocked({ raw: ' anyone around?' })).toBe(true); + expect(await blocked({ raw: ' meeting in five' })).toBe( + true + ); + expect(await blocked({ raw: ' big news' })).toBe(true); + }); + + test('a direct bot mention overrides the ping group', async () => { + expect( + await blocked({ + raw: `<@${BOT_USER_ID}> what do you think?`, + }) + ).toBe(false); + }); + + test('angle-bracket text that is not a ping group passes through', async () => { + expect(await blocked({ raw: ' is not a token' })).toBe(false); + }); +}); + +describe('rule 4: angle-bracket opt-out', () => { + test('ignores messages starting with <>', async () => { + expect(await blocked({ raw: '<> do not parse this' })).toBe(true); + }); + + test('tolerates leading whitespace', async () => { + expect(await blocked({ raw: ' <> do not parse this' })).toBe(true); + }); + + test('a direct bot mention overrides the <> prefix', async () => { + expect( + await blocked({ raw: `<> <@${BOT_USER_ID}> actually parse this` }) + ).toBe(false); + }); + + test('<> later in the text is not an opt-out', async () => { + expect(await blocked({ raw: 'generics look like a <> here' })).toBe(false); + }); +}); + +describe('normal traffic', () => { + test('processes plain messages', async () => { + expect(await blocked({ raw: 'hey can you help me debug this?' })).toBe( + false + ); + }); + + test('processes direct mentions', async () => { + expect(await blocked({ raw: `<@${BOT_USER_ID}> run this` })).toBe(false); + }); + + test('processes mentions of other users', async () => { + expect(await blocked({ raw: '<@UOTHER99999> what do you think?' })).toBe( + false + ); + }); + + test('does not treat a near-miss id as a direct mention', async () => { + expect(await blocked({ raw: `<@${BOT_USER_ID}XX> hi` })).toBe(false); + }); + + test('still blocks ping groups when the bot id is unknown', async () => { + expect( + await blocked({ + botUserId: undefined, + raw: ' ping', + }) + ).toBe(true); + expect(await blocked({ botUserId: undefined, raw: 'plain hello' })).toBe( + false + ); + }); +}); diff --git a/src/mastra/chat/guardrails.ts b/src/mastra/chat/guardrails.ts new file mode 100644 index 0000000..ce38efe --- /dev/null +++ b/src/mastra/chat/guardrails.ts @@ -0,0 +1,63 @@ +import { rawText, withoutLeadingMentions } from './message'; +import { threadState } from './state'; + +// RFC i - Guidelines for AI agents in slack (Hack Club canvas F0BNTDRNL3T). +// All four checks run before command parsing and any LLM work. +interface RawTextSource { + raw: unknown; + text: string; +} + +// Usergroup mentions arrive as (or legacy ) and the +// built-in broadcasts are , , and , optionally +// followed by a |label. +const pingGroupPattern = + /]*>|]*)?>/; + +function isComment(message: RawTextSource): boolean { + for (const line of rawText(message).split('\n')) { + if (withoutLeadingMentions(line).trimStart().startsWith('##')) { + return true; + } + } + return false; +} + +export async function blockedByAgentGuidelines({ + botUserId, + message, + thread, +}: { + botUserId: string | undefined; + message: RawTextSource; + thread: { readonly state: Promise }; +}): Promise { + // Rule 1: ## comments are never processed, even when the bot is mentioned. + if (isComment(message)) { + return true; + } + + // Rule 2: after @gorkie !stop, ignore every later message in the thread. + const state = await threadState(thread); + if (state?.stopped === true) { + return true; + } + + const text = rawText(message); + + // Rules 3 and 4 yield only to an explicit <@USER_ID> token for the bot; a + // bare "@gorkie" typed as plain text does not count as a direct mention. + const directlyMentioned = + botUserId !== undefined && text.includes(`<@${botUserId}>`); + if (directlyMentioned) { + return false; + } + + // Rule 3: a ping group mention is not a bot mention. + if (pingGroupPattern.test(text)) { + return true; + } + + // Rule 4: a leading <> opts the message out unless the bot is mentioned. + return text.trimStart().startsWith('<>'); +} diff --git a/src/mastra/chat/handlers.ts b/src/mastra/chat/handlers.ts index 4dfccb2..00e5ca2 100644 --- a/src/mastra/chat/handlers.ts +++ b/src/mastra/chat/handlers.ts @@ -5,7 +5,7 @@ import { logger } from '../lib/logger'; import { attachments } from './attachments'; import { slack } from './client'; import { handleCommand } from './commands'; -import { rawText, withoutLeadingMentions } from './message'; +import { blockedByAgentGuidelines } from './guardrails'; import { offerOptIn } from './onboarding'; import { threadState } from './state'; @@ -37,15 +37,6 @@ function isFromBot(message: Message): boolean { ); } -function isComment(message: Message): boolean { - for (const line of rawText(message).split('\n')) { - if (withoutLeadingMentions(line).trimStart().startsWith('##')) { - return true; - } - } - return false; -} - async function runTurn({ defaultHandler, message, @@ -79,6 +70,15 @@ export async function onMention( if (isFromBot(message)) { return; } + if ( + await blockedByAgentGuidelines({ + botUserId: slack.botUserId, + message, + thread, + }) + ) { + return; + } if (!(await isUserAllowed(message.author.userId))) { await offerOptIn({ thread, user: message.author }); return; @@ -98,7 +98,16 @@ export async function onSubscribedMessage( defaultHandler: DefaultHandler ): Promise { await captureSearchToken({ raw: message.raw, thread }); - if (isFromBot(message) || isComment(message)) { + if (isFromBot(message)) { + return; + } + if ( + await blockedByAgentGuidelines({ + botUserId: slack.botUserId, + message, + thread, + }) + ) { return; } const state = await threadState(thread); @@ -131,6 +140,15 @@ export async function onDirectMessage( if (isFromBot(message)) { return; } + if ( + await blockedByAgentGuidelines({ + botUserId: slack.botUserId, + message, + thread, + }) + ) { + return; + } if (!(await isUserAllowed(message.author.userId))) { await offerOptIn({ thread, user: message.author }); return; diff --git a/src/mastra/chat/state.ts b/src/mastra/chat/state.ts index 65ee31c..ba12d82 100644 --- a/src/mastra/chat/state.ts +++ b/src/mastra/chat/state.ts @@ -5,6 +5,7 @@ import type { ThreadState } from '../types'; const threadStateSchema = z.looseObject({ respondOnThreadMessages: z.boolean().optional(), searchToken: z.string().optional(), + stopped: z.boolean().optional(), }); export async function threadState( diff --git a/src/mastra/types/thread.ts b/src/mastra/types/thread.ts index b83d74b..0ff9b81 100644 --- a/src/mastra/types/thread.ts +++ b/src/mastra/types/thread.ts @@ -1,4 +1,5 @@ export interface ThreadState { respondOnThreadMessages?: boolean; searchToken?: string; + stopped?: boolean; }