AI: Declare tools unconditionally for imported/non-fresh files

Instead of removing/hiding page-touching tools entirely when dealing
with imported Lighthouse reports or non-fresh traces, keep them declared.
When AIDA attempts to invoke them, early exit inside their handlers
with a concise error message indicating they cannot be used.

This also removes the security warning prepended to the prompt
preambles, simplifying the initial query.

Fixed: 513711812
Change-Id: Ia680e27131d34a6ec97ca9b6bc0e45145af0a27e
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/7912246
Auto-Submit: Jack Franklin <jacktfranklin@chromium.org>
Commit-Queue: Jack Franklin <jacktfranklin@chromium.org>
Reviewed-by: Paul Irish <paulirish@chromium.org>

Author: jackfranklin

front_end/models/ai_assistance/agents/AccessibilityAgent.test.ts

@@ -417,14 +417,79 @@ describeWithMockConnection('AccessibilityAgent', () => {
     sinon.stub(domModel, 'existingDocument').returns(document);
 
     const responses = await Array.fromAsync(agent.run('test', {selected: context}));
-    const errorResponse = responses.find(response => response.type === AiAssistance.AiAgent.ResponseType.ERROR);
-    assert.exists(errorResponse);
-    if (errorResponse && 'error' in errorResponse) {
-      assert.strictEqual(errorResponse.error, AiAssistance.AiAgent.ErrorType.UNKNOWN);
-    }
+    const actionResponse = responses.find(response => response.type === AiAssistance.AiAgent.ResponseType.ACTION);
+    assert.exists(actionResponse);
+    assert.strictEqual(actionResponse.output, 'Cannot use this tool on an imported file.');
     sinon.assert.notCalled(execJs);
   });
 
+  it('cannot call runAccessibilityAudits if the report is imported', async () => {
+    const aidaClient = mockAidaClient([[{
+      explanation: '',
+      functionCalls: [{name: 'runAccessibilityAudits', args: {explanation: 'testing'}}],
+    }]]);
+    const lighthouseRecording = sinon.stub().resolves(mockReport);
+    const agent = new AiAssistance.AccessibilityAgent.AccessibilityAgent({
+      aidaClient,
+      lighthouseRecording,
+    });
+
+    const importedReport: LHModel.ReporterTypes.ReportJSON = {
+      ...mockReport,
+      isImported: true,
+    };
+    const context = new AiAssistance.AccessibilityAgent.AccessibilityContext(importedReport);
+
+    const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+    const actionResponse = responses.find(response => response.type === AiAssistance.AiAgent.ResponseType.ACTION);
+    assert.exists(actionResponse);
+    assert.strictEqual(actionResponse.output, 'Cannot use this tool on an imported file.');
+    sinon.assert.notCalled(lighthouseRecording);
+  });
+
+  it('cannot call getStyles if the report is imported', async () => {
+    const aidaClient = mockAidaClient([[{
+      explanation: '',
+      functionCalls:
+          [{name: 'getStyles', args: {path: '1,HTML,1,BODY', styleProperties: ['color'], explanation: 'testing'}}],
+    }]]);
+    const agent = new AiAssistance.AccessibilityAgent.AccessibilityAgent({
+      aidaClient,
+    });
+
+    const importedReport: LHModel.ReporterTypes.ReportJSON = {
+      ...mockReport,
+      isImported: true,
+    };
+    const context = new AiAssistance.AccessibilityAgent.AccessibilityContext(importedReport);
+
+    const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+    const actionResponse = responses.find(response => response.type === AiAssistance.AiAgent.ResponseType.ACTION);
+    assert.exists(actionResponse);
+    assert.strictEqual(actionResponse.output, 'Cannot use this tool on an imported file.');
+  });
+
+  it('cannot call getElementAccessibilityDetails if the report is imported', async () => {
+    const aidaClient = mockAidaClient([[{
+      explanation: '',
+      functionCalls: [{name: 'getElementAccessibilityDetails', args: {path: '1,HTML,1,BODY', explanation: 'testing'}}],
+    }]]);
+    const agent = new AiAssistance.AccessibilityAgent.AccessibilityAgent({
+      aidaClient,
+    });
+
+    const importedReport: LHModel.ReporterTypes.ReportJSON = {
+      ...mockReport,
+      isImported: true,
+    };
+    const context = new AiAssistance.AccessibilityAgent.AccessibilityContext(importedReport);
+
+    const responses = await Array.fromAsync(agent.run('test', {selected: context}));
+    const actionResponse = responses.find(response => response.type === AiAssistance.AiAgent.ResponseType.ACTION);
+    assert.exists(actionResponse);
+    assert.strictEqual(actionResponse.output, 'Cannot use this tool on an imported file.');
+  });
+
   it('can still call getLighthouseAudits if the report is imported', async () => {
     const aidaClient = mockAidaClient([[{
       explanation: '',
@@ -453,33 +518,14 @@ describeWithMockConnection('AccessibilityAgent', () => {
   });
 
   describe('enhanceQuery', () => {
-    it('prepends security warning if the report is imported', async () => {
-      const agent = new AiAssistance.AccessibilityAgent.AccessibilityAgent({
-        aidaClient: mockAidaClient([]),
-      });
-
-      const importedReport: LHModel.ReporterTypes.ReportJSON = {
-        ...mockReport,
-        isImported: true,
-      };
-      const context = new AiAssistance.AccessibilityAgent.AccessibilityContext(importedReport);
-
-      const enhancedQuery = await agent.enhanceQuery('user query', context);
-
-      assert.include(enhancedQuery, 'imported from a file and is static');
-      assert.include(enhancedQuery, 'user query');
-    });
-
-    it('does not prepend security warning if the report is not imported', async () => {
+    it('adds the context to the query', async () => {
       const agent = new AiAssistance.AccessibilityAgent.AccessibilityAgent({
         aidaClient: mockAidaClient([]),
       });
 
       const context = new AiAssistance.AccessibilityAgent.AccessibilityContext(mockReport);
-
       const enhancedQuery = await agent.enhanceQuery('user query', context);
-
-      assert.notInclude(enhancedQuery, 'imported from a file and is static');
+      assert.include(enhancedQuery, '# Lighthouse Report');
       assert.include(enhancedQuery, 'user query');
     });
   });

front_end/models/ai_assistance/agents/AccessibilityAgent.ts

@@ -81,12 +81,6 @@ If the user asks a question that requires an investigation of a problem, use thi
     - [Suggestion 2]
 `;
 
-const SECURITY_WARNING = `**CRITICAL CONSTRAINT**: This Lighthouse report was imported from a file and is static.
-You do NOT have access to the inspected page.
-Tools like \`executeJavaScript\`, \`getStyles\`, or \`runAccessibilityAudits\` are disabled.
-Do NOT attempt to use them or instruct the user that you will use them.
-Rely ONLY on the static report data below.`;
-
 export class AccessibilityContext extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
   #lh: LHModel.ReporterTypes.ReportJSON;
 
@@ -277,11 +271,18 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
       }
     });
 
-    if (isImported) {
-      return;
-    }
-
-    this.declareFunction('executeJavaScript', executeJavaScriptFunction(this.#javascriptExecutor));
+    const executeJsDeclaration = executeJavaScriptFunction(this.#javascriptExecutor);
+    this.declareFunction('executeJavaScript', {
+      ...executeJsDeclaration,
+      handler: async (params, options) => {
+        if (isImported) {
+          return {
+            error: 'Cannot use this tool on an imported file.',
+          };
+        }
+        return await executeJsDeclaration.handler(params, options);
+      },
+    });
 
     this.declareFunction<{explanation: string}, {audits: string}>('runAccessibilityAudits', {
       description:
@@ -308,6 +309,11 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
       },
       handler: async params => {
         debugLog('Function call: runAccessibilityAudits', params);
+        if (isImported) {
+          return {
+            error: 'Cannot use this tool on an imported file.',
+          };
+        }
         if (!this.#lighthouseRecording) {
           return {error: 'Lighthouse recording is not available.'};
         }
@@ -372,6 +378,11 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
       },
       handler: async params => {
         debugLog('Function call: getStyles', params);
+        if (isImported) {
+          return {
+            error: 'Cannot use this tool on an imported file.',
+          };
+        }
         const node = await this.#resolvePathToNode(params.path);
         if (!node) {
           return {error: `Could not find the element with path: ${params.path}`};
@@ -442,6 +453,11 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
       },
       handler: async params => {
         debugLog('Function call: getElementAccessibilityDetails', params);
+        if (isImported) {
+          return {
+            error: 'Cannot use this tool on an imported file.',
+          };
+        }
         const node = await this.#resolvePathToNode(params.path);
         if (!node) {
           return {error: `Could not find the element with path: ${params.path}`};
@@ -519,10 +535,7 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
     if (lhr) {
       this.#declareFunctions();
     }
-    let enhancedQuery = lhr ? `${this.#getInitialPayload(lhr)}\n# User request:\n\n` : '';
-    if (lhr?.getItem().isImported) {
-      enhancedQuery = `${SECURITY_WARNING}\n\n${enhancedQuery}`;
-    }
+    const enhancedQuery = lhr ? `${this.#getInitialPayload(lhr)}\n# User request:\n\n` : '';
     return `${enhancedQuery}${query}`;
   }
 

front_end/models/ai_assistance/agents/PerformanceAgent.test.ts

@@ -663,11 +663,9 @@ code
       sinon.stub(Tracing.FreshRecording.Tracker.instance(), 'recordingIsFresh').returns(false);
 
       const responses = await Array.fromAsync(agent.run('test', {selected: context}));
-      const errorResponse = responses.find(response => response.type === AiAgent.ResponseType.ERROR);
-      assert.exists(errorResponse);
-      if (errorResponse && 'error' in errorResponse) {
-        assert.strictEqual(errorResponse.error, AiAgent.ErrorType.UNKNOWN);
-      }
+      const actionResponse = responses.find(response => response.type === AiAgent.ResponseType.ACTION);
+      assert.exists(actionResponse);
+      assert.strictEqual(actionResponse.output, 'Cannot use this tool on an imported file.');
     });
 
     it('can call getMainThreadTrackSummaryByLabel', async function() {