Skip to content

Commit 30a943f

Browse files
vtempestvtempest
committed
feat: add URL scraping functionality and YouTube transcript extraction
- Implemented `scrapeURL` function for fetching HTML content from any URL with bot detection handling and fallback mechanisms. - Created `youtube-helpers.ts` for extracting YouTube video IDs and fetching transcripts using the `extract-youtube` library. - Developed `youtube-to-text.ts` to convert YouTube video pages into text transcripts, including timestamp handling and optional player embedding. - Added support for fetching transcripts from various sources, including official YouTube captions and third-party services.
1 parent 792a602 commit 30a943f

9 files changed

Lines changed: 3002 additions & 0 deletions

File tree

‎apps/qwksearch-web/vite.config.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,18 @@ export default defineConfig(({ command }) => ({
4040
external: ["fsevents"],
4141
},
4242
},
43+
ssr: {
44+
// Bundle workspace packages into the standalone output instead of treating
45+
// them as external dependencies (which vinext can't resolve at deploy time)
46+
noExternal: [
47+
"chat-agent-toolkit",
48+
"extract-webpage",
49+
"extract-pdf",
50+
"extract-youtube",
51+
"qwksearch-api-client",
52+
"shadcn-app-dock",
53+
],
54+
},
4355
plugins: [
4456
{
4557
// Resolve `cloudflare:workers` to a harmless stub in the client build,

‎packages/agent-toolkit/src/extractor/url-to-content/__tests__/url-to-content.test.ts‎

Lines changed: 368 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,368 @@
1+
/**
2+
* @fileoverview Unit tests for URL content extraction
3+
*/
4+
import { extractContent } from "../url-to-content";
5+
import { scrapeURL } from "../url-to-html";
6+
import { extractContentAndCite } from "../../html-to-content/html-to-content";
7+
import { convertYoutubeToText } from "../youtube-helpers";
8+
import { convertPDFToHTML } from "extract-pdf";
9+
10+
// Mock dependencies
11+
jest.mock("../url-to-html", () => ({
12+
scrapeURL: jest.fn(),
13+
}));
14+
15+
jest.mock("../../html-to-content/html-to-content", () => ({
16+
extractContentAndCite: jest.fn(),
17+
}));
18+
19+
jest.mock("../youtube-helpers", () => ({
20+
getURLYoutubeVideo: jest.fn(),
21+
convertYoutubeToText: jest.fn(),
22+
}));
23+
24+
jest.mock("extract-pdf", () => ({
25+
convertPDFToHTML: jest.fn(),
26+
}));
27+
28+
jest.mock("grab-url", () => jest.fn());
29+
30+
const mockScrapeURL = scrapeURL as jest.MockedFunction<typeof scrapeURL>;
31+
const mockExtractContentAndCite = extractContentAndCite as jest.MockedFunction<
32+
typeof extractContentAndCite
33+
>;
34+
const mockConvertYoutubeToText =
35+
convertYoutubeToText as jest.MockedFunction<typeof convertYoutubeToText>;
36+
const mockConvertPDFToHTML = convertPDFToHTML as jest.MockedFunction<
37+
typeof convertPDFToHTML
38+
>;
39+
40+
describe("extractContent", () => {
41+
beforeEach(() => {
42+
jest.clearAllMocks();
43+
});
44+
45+
describe("URL extraction", () => {
46+
it("should extract content from a regular URL", async () => {
47+
const mockHtml = "<html><body><article>Test content</article></body></html>";
48+
const mockExtracted = {
49+
title: "Test Article",
50+
html: "<p>Test content</p>",
51+
author: "John Doe",
52+
date: "2024-01-01",
53+
source: "Example",
54+
word_count: 2,
55+
};
56+
57+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
58+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
59+
60+
const result = await extractContent("https://example.com/article");
61+
62+
expect(mockScrapeURL).toHaveBeenCalledWith("https://example.com/article", {
63+
proxy: null,
64+
});
65+
expect(mockExtractContentAndCite).toHaveBeenCalledWith(mockHtml, {
66+
url: "https://example.com/article",
67+
images: true,
68+
links: true,
69+
formatting: true,
70+
absoluteURLs: true,
71+
timeout: 10,
72+
proxy: null,
73+
citeFormatMonthFull: false,
74+
citeFormatAuthorFull: true,
75+
});
76+
expect(result.title).toBe("Test Article");
77+
expect(result.html).toBe("<p>Test content</p>");
78+
});
79+
80+
it("should handle scrapeURL returning error object", async () => {
81+
mockScrapeURL.mockResolvedValueOnce({
82+
error: "HTTP error: 403 Forbidden",
83+
} as any);
84+
85+
const result = await extractContent("https://blocked-site.com/article");
86+
87+
expect(result.error).toBe("HTTP error: 403 Forbidden");
88+
expect(mockExtractContentAndCite).not.toHaveBeenCalled();
89+
});
90+
91+
it("should handle scrapeURL returning null", async () => {
92+
mockScrapeURL.mockResolvedValueOnce(null as any);
93+
94+
const result = await extractContent("https://null-response.com/article");
95+
96+
expect(result.error).toBe("Failed to fetch HTML content");
97+
expect(mockExtractContentAndCite).not.toHaveBeenCalled();
98+
});
99+
100+
it("should handle scrapeURL returning undefined", async () => {
101+
mockScrapeURL.mockResolvedValueOnce(undefined as any);
102+
103+
const result = await extractContent(
104+
"https://undefined-response.com/article"
105+
);
106+
107+
expect(result.error).toBe("Failed to fetch HTML content");
108+
expect(mockExtractContentAndCite).not.toHaveBeenCalled();
109+
});
110+
111+
it("should handle scrapeURL returning empty string", async () => {
112+
mockScrapeURL.mockResolvedValueOnce("");
113+
114+
const result = await extractContent("https://empty-response.com/article");
115+
116+
expect(result.error).toBe("Failed to fetch HTML content");
117+
expect(mockExtractContentAndCite).not.toHaveBeenCalled();
118+
});
119+
120+
it("should handle scrapeURL throwing an error", async () => {
121+
mockScrapeURL.mockRejectedValueOnce(new Error("Network timeout"));
122+
123+
await expect(
124+
extractContent("https://timeout.com/article")
125+
).rejects.toThrow("Network timeout");
126+
});
127+
128+
it("should handle extraction returning no HTML", async () => {
129+
const mockHtml = "<html><body>Content</body></html>";
130+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
131+
mockExtractContentAndCite.mockReturnValueOnce({
132+
title: "Article",
133+
html: "", // Empty HTML
134+
} as any);
135+
136+
const result = await extractContent("https://example.com/empty");
137+
138+
expect(result.error).toBeUndefined(); // This is actually a success case with empty content
139+
});
140+
141+
it("should extract from raw HTML string", async () => {
142+
const mockHtml = "<html><body><p>Test content</p></body></html>";
143+
const mockExtracted = {
144+
title: "Test",
145+
html: "<p>Test content</p>",
146+
word_count: 2,
147+
};
148+
149+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
150+
151+
const result = await extractContent(mockHtml, { url: "https://example.com" });
152+
153+
expect(mockScrapeURL).not.toHaveBeenCalled();
154+
expect(mockExtractContentAndCite).toHaveBeenCalledWith(mockHtml, expect.any(Object));
155+
expect(result.title).toBe("Test");
156+
});
157+
158+
it("should handle extraction with custom options", async () => {
159+
const mockHtml = "<html><body>Content</body></html>";
160+
const mockExtracted = {
161+
title: "Test",
162+
html: "<p>Content</p>",
163+
word_count: 1,
164+
};
165+
166+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
167+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
168+
169+
await extractContent("https://example.com/article", {
170+
images: false,
171+
links: false,
172+
formatting: false,
173+
timeout: 15,
174+
proxy: "https://proxy.example.com",
175+
});
176+
177+
expect(mockScrapeURL).toHaveBeenCalledWith("https://example.com/article", {
178+
proxy: "https://proxy.example.com",
179+
});
180+
expect(mockExtractContentAndCite).toHaveBeenCalledWith(
181+
mockHtml,
182+
expect.objectContaining({
183+
images: false,
184+
links: false,
185+
formatting: false,
186+
timeout: 15,
187+
proxy: "https://proxy.example.com",
188+
})
189+
);
190+
});
191+
});
192+
193+
describe("YouTube extraction", () => {
194+
it("should extract YouTube video transcript", async () => {
195+
const youtubeHelpers = require("../youtube-helpers");
196+
youtubeHelpers.getURLYoutubeVideo.mockReturnValueOnce("dQw4w9WgXcQ");
197+
198+
const mockTranscript = {
199+
title: "Test Video",
200+
html: "<p>Transcript content</p>",
201+
source: "YouTube",
202+
word_count: 2,
203+
};
204+
205+
mockConvertYoutubeToText.mockResolvedValueOnce(mockTranscript);
206+
207+
const result = await extractContent(
208+
"https://www.youtube.com/watch?v=dQw4w9WgXcQ"
209+
);
210+
211+
expect(mockConvertYoutubeToText).toHaveBeenCalledWith(
212+
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
213+
expect.any(Object)
214+
);
215+
expect(result.title).toBe("Test Video");
216+
expect(result.source).toBe("YouTube");
217+
expect(mockScrapeURL).not.toHaveBeenCalled();
218+
});
219+
});
220+
221+
describe("PDF extraction", () => {
222+
it("should extract content from PDF URLs", async () => {
223+
const mockPdfContent = {
224+
title: "PDF Document",
225+
html: "<p>PDF content</p>",
226+
word_count: 2,
227+
};
228+
229+
mockConvertPDFToHTML.mockResolvedValueOnce(mockPdfContent as any);
230+
231+
const result = await extractContent("https://example.com/document.pdf");
232+
233+
expect(mockConvertPDFToHTML).toHaveBeenCalledWith(
234+
"https://example.com/document.pdf",
235+
expect.any(Object)
236+
);
237+
expect(result.title).toBe("PDF Document");
238+
expect(mockScrapeURL).not.toHaveBeenCalled();
239+
});
240+
});
241+
242+
describe("Google Docs extraction", () => {
243+
it("should rewrite Google Doc URLs to export format", async () => {
244+
const mockHtml = "<html><body>Google Doc content</body></html>";
245+
const mockExtracted = {
246+
title: "Google Doc",
247+
html: "<p>Content</p>",
248+
word_count: 1,
249+
};
250+
251+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
252+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
253+
254+
await extractContent(
255+
"https://docs.google.com/document/d/ABC123/edit"
256+
);
257+
258+
expect(mockScrapeURL).toHaveBeenCalledWith(
259+
"https://docs.google.com/document/d/ABC123/export?format=html",
260+
expect.any(Object)
261+
);
262+
});
263+
264+
it("should rewrite Google Drive file URLs", async () => {
265+
const mockPdfContent = {
266+
title: "Drive PDF",
267+
html: "<p>PDF from Drive</p>",
268+
word_count: 3,
269+
};
270+
271+
mockConvertPDFToHTML.mockResolvedValueOnce(mockPdfContent as any);
272+
273+
await extractContent(
274+
"https://drive.google.com/file/d/ABC123/view"
275+
);
276+
277+
expect(mockConvertPDFToHTML).toHaveBeenCalledWith(
278+
"https://drive.google.com/uc?export=download&id=ABC123",
279+
expect.any(Object)
280+
);
281+
});
282+
});
283+
284+
describe("Error handling", () => {
285+
it("should return error if extraction returns error", async () => {
286+
const mockHtml = "<html><body>Content</body></html>";
287+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
288+
mockExtractContentAndCite.mockReturnValueOnce({
289+
error: "Failed to parse content",
290+
} as any);
291+
292+
const result = await extractContent("https://example.com/article");
293+
294+
expect(result.error).toBe("Failed to parse content");
295+
});
296+
297+
it("should return error for invalid input type", async () => {
298+
const result = await extractContent({ invalid: "object" } as any);
299+
300+
expect(result.error).toContain("Invalid input type");
301+
});
302+
});
303+
304+
describe("Word count and citation", () => {
305+
it("should calculate word count from HTML", async () => {
306+
const mockHtml = "<html><body>Content</body></html>";
307+
const mockExtracted = {
308+
title: "Test Article",
309+
html: "<p>This is a test article with some words</p>",
310+
author_cite: "Doe, J.",
311+
date: "2024-01-15",
312+
source: "Example",
313+
};
314+
315+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
316+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
317+
318+
const result = await extractContent("https://example.com/article");
319+
320+
expect(result.word_count).toBeGreaterThan(0);
321+
});
322+
323+
it("should generate APA citation", async () => {
324+
const mockHtml = "<html><body>Content</body></html>";
325+
const mockExtracted = {
326+
title: "Test Article",
327+
html: "<p>Content</p>",
328+
author_cite: "Smith, J.",
329+
date: "2024-01-15",
330+
source: "Example News",
331+
word_count: 1,
332+
};
333+
334+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
335+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
336+
337+
const result = await extractContent("https://example.com/article");
338+
339+
expect(result.cite).toContain("Smith, J.");
340+
expect(result.cite).toContain("(2024");
341+
expect(result.cite).toContain("Test Article");
342+
expect(result.cite).toContain("Example News");
343+
});
344+
345+
it("should shorten long URLs in citation", async () => {
346+
const longUrl =
347+
"https://example.com/article?" +
348+
"param1=value1&param2=value2&param3=value3&" +
349+
"tracking=12345&sessionid=abcdefghijklmnopqrstuvwxyz&" +
350+
"utm_source=test&utm_medium=test&utm_campaign=test";
351+
352+
const mockHtml = "<html><body>Content</body></html>";
353+
const mockExtracted = {
354+
title: "Article",
355+
html: "<p>Content</p>",
356+
word_count: 1,
357+
};
358+
359+
mockScrapeURL.mockResolvedValueOnce(mockHtml);
360+
mockExtractContentAndCite.mockReturnValueOnce(mockExtracted);
361+
362+
const result = await extractContent(longUrl);
363+
364+
expect(result.url).toBe("https://example.com/article");
365+
expect(result.url).not.toContain("?");
366+
});
367+
});
368+
});

0 commit comments

Comments
 (0)