#18 support youtube/bilibili videos

Author: chaoyuepan

README.md

@@ -21,7 +21,7 @@ An AI-powered knowledge management application that lets you create intelligent
 
 ## ✨ Features
 
-- 📚 **Multiple Source Types** - Upload PDFs, text files, Markdown, DOCX, and HTML documents
+- 📚 **Multiple Source Types** - Upload PDFs, text files, Markdown, DOCX, HTML documents, and video URLs (YouTube, Bilibili with automatic subtitle extraction)
 - 🤖 **AI-Powered Chat** - Ask questions and get answers based on your sources
 - ✨ **Multiple Transformations** - Generate summaries, FAQs, study guides, outlines, timelines, glossaries, quizzes, mindmaps, infographics and podcast scripts
 - 📊 **Infographic Generation** - Create beautiful, hand-drawn style infographics from your content using Google's Gemini Nano Banana
@@ -36,6 +36,8 @@ An AI-powered knowledge management application that lets you create intelligent
 
 - Go 1.23 or later
 - An LLM API key (OpenAI) or Ollama running locally
+- [markitdown](https://github.com/microsoft/markitdown) (optional, for better document conversion)
+- [yt-dlp](https://github.com/yt-dlp/yt-dlp) (optional, for extracting subtitles from YouTube and Bilibili videos)
 
 ### Installation
 
@@ -179,6 +181,8 @@ You can add content to your notebook in three ways:
 
 - Select the "URL" tab
 - Enter the URL and optional title
+- Supports web pages and video URLs (YouTube, Bilibili)
+- For videos, subtitles are automatically extracted and used as content
 
 ### Chatting with Sources
 

README_CN.md

@@ -21,7 +21,7 @@ AI 驱动知识管理应用程序，让您从文档中创建智能笔记本。
 
 ## ✨ 特性
 
-- 📚 **多种来源类型** - 支持上传 PDF、文本文件、Markdown、DOCX 和 HTML 文档
+- 📚 **多种来源类型** - 支持上传 PDF、文本文件、Markdown、DOCX、HTML 文档和视频 URL（YouTube、Bilibili 自动提取字幕）
 - 🤖 **AI 驱动对话** - 基于您的来源提问并获得答案
 - ✨ **多种转换** - 生成摘要、FAQ、学习指南、大纲、时间线、词汇表、测验、思维导图、信息图和播客脚本
 - 📊 **信息图生成** - 使用 Google Gemini Nano Banana 从您的内容创建精美的手绘风格信息图
@@ -36,6 +36,8 @@ AI 驱动知识管理应用程序，让您从文档中创建智能笔记本。
 
 - Go 1.23 或更高版本
 - LLM API 密钥 (OpenAI) 或本地运行的 Ollama
+- [markitdown](https://github.com/microsoft/markitdown)（可选，用于更好的文档转换）
+- [yt-dlp](https://github.com/yt-dlp/yt-dlp)（可选，用于从 YouTube 和 Bilibili 视频提取字幕）
 
 ### 安装
 
@@ -179,6 +181,8 @@ go build -o notex .
 
 - 选择 "URL" 标签
 - 输入 URL 和可选标题
+- 支持网页和视频 URL（YouTube、Bilibili）
+- 对于视频，会自动提取字幕作为内容
 
 ### 与来源对话
 

backend/vector.go

@@ -350,6 +350,141 @@ func (vs *VectorStore) needsMarkitdown(ext string) bool {
 	return markitdownExts[ext]
 }
 
+
+// isVideoURL checks if the URL is from a video platform (YouTube, Bilibili)
+func isVideoURL(url string) bool {
+	patterns := []string{
+		"youtube.com",
+		"youtu.be",
+		"bilibili.com",
+		"b23.tv",
+	}
+	lowerURL := strings.ToLower(url)
+	for _, pattern := range patterns {
+		if strings.Contains(lowerURL, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// extractVideoSubtitle extracts subtitle from video URL using yt-dlp
+func (vs *VectorStore) extractVideoSubtitle(url string) (string, error) {
+	fmt.Printf("[VectorStore] Extracting subtitle from video: %s\n", url)
+
+	// Create temporary file for subtitle
+	tmpSubFile := filepath.Join(os.TempDir(), fmt.Sprintf("subtitle_%d.srt", os.Getpid()))
+
+	// Use yt-dlp to download subtitles
+	// --write-subs: write subtitles
+	// --sub-langs all: download all available languages
+	// --skip-download: don't download the video itself
+	// --sub-format srt: use SRT format
+	// -o: output file
+	cmd := exec.Command("yt-dlp",
+		"--write-subs",
+		"--sub-langs", "all",
+		"--skip-download",
+		"--sub-format", "srt",
+		"-o", tmpSubFile,
+		url,
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		fmt.Printf("[VectorStore] yt-dlp error: %s\n", string(output))
+		return "", fmt.Errorf("failed to extract subtitle: %w, output: %s", err, string(output))
+	}
+
+	// Find the actual subtitle file (yt-dlp may add language suffix)
+	matches, _ := filepath.Glob(tmpSubFile + "*.srt")
+	if len(matches) == 0 {
+		return "", fmt.Errorf("no subtitle file found")
+	}
+	actualSubFile := matches[0]
+
+	// Read the subtitle file
+	subContent, err := os.ReadFile(actualSubFile)
+	if err != nil {
+		os.Remove(actualSubFile)
+		return "", fmt.Errorf("failed to read subtitle file: %w", err)
+	}
+
+	// Clean up subtitle file
+	os.Remove(actualSubFile)
+
+	// Convert SRT to plain text format
+	textContent := vs.convertSRTToText(string(subContent))
+
+	fmt.Printf("[VectorStore] Subtitle extracted successfully, size: %d bytes\n", len(textContent))
+	return textContent, nil
+}
+
+// convertSRTToText converts SRT subtitle format to readable text
+func (vs *VectorStore) convertSRTToText(srtContent string) string {
+	lines := strings.Split(srtContent, "\n")
+	var result []string
+	currentText := []string{}
+
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		
+		// Skip empty lines
+		if line == "" {
+			if len(currentText) > 0 {
+				result = append(result, strings.Join(currentText, " "))
+				currentText = []string{}
+			}
+			continue
+		}
+
+		// Skip sequence numbers (1, 2, 3, ...)
+		if _, err := fmt.Sscanf(line, "%d", new(int)); err == nil && len(line) < 10 {
+			if len(currentText) > 0 {
+				result = append(result, strings.Join(currentText, " "))
+				currentText = []string{}
+			}
+			continue
+		}
+
+		// Skip timestamp lines (00:00:00,000 --> 00:00:05,000)
+		if strings.Contains(line, "-->") {
+			if len(currentText) > 0 {
+				result = append(result, strings.Join(currentText, " "))
+				currentText = []string{}
+			}
+			continue
+		}
+
+		// Skip hex/hash lines (sometimes in SRT files)
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		// Remove HTML tags and common subtitle artifacts
+		line = strings.ReplaceAll(line, "<i>", "")
+		line = strings.ReplaceAll(line, "</i>", "")
+		line = strings.ReplaceAll(line, "<b>", "")
+		line = strings.ReplaceAll(line, "</b>", "")
+		line = strings.ReplaceAll(line, "&lt;", "<")
+		line = strings.ReplaceAll(line, "&gt;", ">")
+		line = strings.ReplaceAll(line, "&amp;", "&")
+		line = strings.ReplaceAll(line, "&#39;", "'")
+		line = strings.ReplaceAll(line, "&quot;", "\"")
+
+		// Add to current text if it's not empty
+		if line != "" {
+			currentText = append(currentText, line)
+		}
+	}
+
+	// Don't forget the last segment
+	if len(currentText) > 0 {
+		result = append(result, strings.Join(currentText, " "))
+	}
+
+	return strings.Join(result, "\n")
+}
 // ExtractFromURL fetches and converts content from a URL using markitdown
 func (vs *VectorStore) ExtractFromURL(ctx context.Context, url string) (string, error) {
 	fmt.Printf("[VectorStore] Fetching content from URL: %s\n", url)
@@ -358,6 +493,11 @@ func (vs *VectorStore) ExtractFromURL(ctx context.Context, url string) (string,
 		return "", fmt.Errorf("markitdown is disabled, cannot fetch URL content")
 	}
 
+	// Check if it's a video URL and extract subtitles
+	if isVideoURL(url) {
+		return vs.extractVideoSubtitle(url)
+	}
+
 	// Step 1: Use curl to download the webpage to a temporary HTML file
 	tmpHTMLFile := filepath.Join(os.TempDir(), fmt.Sprintf("webpage_%d.html", os.Getpid()))
 	tmpMDFile := filepath.Join(os.TempDir(), fmt.Sprintf("markitdown_url_%d.md", os.Getpid()))
@@ -398,8 +538,6 @@ func (vs *VectorStore) ExtractFromURL(ctx context.Context, url string) (string,
 	fmt.Printf("[VectorStore] URL content fetched and converted successfully, output size: %d bytes\n", len(content))
 	return string(content), nil
 }
-
-// convertWithMarkitdown converts a document to Markdown using the markitdown CLI tool
 func (vs *VectorStore) convertWithMarkitdown(filePath string) (string, error) {
 	fmt.Printf("[VectorStore] Converting with markitdown: %s\n", filePath)