fix(web): attribute a web source only when it survived into the prompt

Author: kfaracik

__tests__/messageSources.test.ts

@@ -410,6 +410,43 @@ describe('web search results (experimental)', () => {
     expect(usedByName['Cooking recipes']).toBe(false);
   });
 
+  it('does not flag a web result used when it was truncated out of the prompt', () => {
+    const cited = [
+      web(
+        'Weather Warsaw',
+        'https://w.com',
+        'Warsaw temperature and rain forecast today'
+      ),
+    ];
+    const answer =
+      'The weather in Warsaw shows rain and a low temperature today.';
+
+    const result = pickCitationsByAnswer(cited, answer, [], new Set<string>());
+
+    expect(result[0].used).toBe(false);
+  });
+
+  it('flags a web result used only when its name is present in the prompt', () => {
+    const cited = [
+      web(
+        'Weather Warsaw',
+        'https://w.com',
+        'Warsaw temperature and rain forecast today'
+      ),
+    ];
+    const answer =
+      'The weather in Warsaw shows rain and a low temperature today.';
+
+    const result = pickCitationsByAnswer(
+      cited,
+      answer,
+      [],
+      new Set(['Weather Warsaw'])
+    );
+
+    expect(result[0].used).toBe(true);
+  });
+
   it('marks no web result used when the answer is a refusal', () => {
     const cited = [
       web('Weather Warsaw', 'https://w.com', 'Warsaw temperature and rain'),

hooks/useMessageSources.ts

@@ -8,7 +8,10 @@ export const useMessageSources = (sourceDocuments?: SourceDocument[]) => {
 
     const seen = new Set<string>();
     return sourceDocuments.filter((source) => {
-      const key = sourceKey(source.documentId, source.name);
+      const key =
+        source.kind === 'web' && source.url
+          ? `web:${source.url}`
+          : sourceKey(source.documentId, source.name);
       if (seen.has(key)) return false;
       seen.add(key);
       return true;

utils/messageSources.ts

@@ -169,7 +169,8 @@ export const answerCitationOverlaps = (
 export const pickCitationsByAnswer = (
   sourceDocuments: SourceDocument[],
   answer: string,
-  preferred: SourceDocument[]
+  preferred: SourceDocument[],
+  presentNames?: Set<string>
 ): SourceDocument[] => {
   const webDocuments = sourceDocuments.filter(
     (doc) => sourceKind(doc) === 'web'
@@ -182,12 +183,16 @@ export const pickCitationsByAnswer = (
     answer,
     preferred
   );
-  return [...citedLocal, ...flagUsedWebDocuments(webDocuments, answer)];
+  return [
+    ...citedLocal,
+    ...flagUsedWebDocuments(webDocuments, answer, presentNames),
+  ];
 };
 
 const flagUsedWebDocuments = (
   webDocuments: SourceDocument[],
-  answer: string
+  answer: string,
+  presentNames?: Set<string>
 ): SourceDocument[] => {
   if (webDocuments.length === 0) return webDocuments;
 
@@ -205,7 +210,9 @@ const flagUsedWebDocuments = (
   return scored.map((s) => ({
     ...s.doc,
     used:
-      maxOverlap > 0 && s.overlap >= maxOverlap * ANSWER_CITATION_OVERLAP_RATIO,
+      maxOverlap > 0 &&
+      s.overlap >= maxOverlap * ANSWER_CITATION_OVERLAP_RATIO &&
+      (presentNames === undefined || presentNames.has(s.doc.name)),
   }));
 };