(#271) Semantic/Vector Search

Author: phongnguyend

src/Monolith/ClassifiedAds.Background/MessageBusConsumers/FileEmbeddingConsumer.cs

@@ -10,12 +10,15 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Security.Cryptography;
+using System.Text;
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -64,7 +67,17 @@ public Task HandleAsync(FileDeletedEvent data, MetaData metaData, CancellationTo
 
     private async Task ProcessFileAsync(FileEntry fileEntry, CancellationToken cancellationToken)
     {
-        if (string.IsNullOrEmpty(fileEntry?.FileLocation))
+        if (fileEntry == null)
+        {
+            return;
+        }
+
+        if (string.IsNullOrEmpty(fileEntry.FileLocation))
+        {
+            return;
+        }
+
+        if (fileEntry.Encrypted && string.IsNullOrEmpty(fileEntry.EncryptionKey))
         {
             return;
         }
@@ -78,8 +91,22 @@ private async Task ProcessFileAsync(FileEntry fileEntry, CancellationToken cance
            fileExtension == ".md" ||
            fileExtension == ".markdown")
         {
-            // TODO: xxx
-            return;
+
+            var bytes = await GetBytesAsync(fileStorageManager, fileEntry, cancellationToken);
+
+            var chunks = TextChunkingService.ChunkSentences(Encoding.UTF8.GetString(bytes));
+
+            var chunksFolder = Path.Combine(_configuration["Storage:TempFolderPath"], "Chunks", fileEntry.Id.ToString());
+
+            if (!Directory.Exists(chunksFolder))
+            {
+                Directory.CreateDirectory(chunksFolder);
+            }
+
+            foreach (var chunk in chunks)
+            {
+                await File.WriteAllTextAsync(Path.Combine(chunksFolder, $"{chunk.StartIndex}_{chunk.EndIndex}.txt"), chunk.Text, cancellationToken);
+            }
         }
         else if (fileExtension == ".pdf" ||
             fileExtension == ".docx" ||
@@ -101,6 +128,20 @@ private async Task ProcessFileAsync(FileEntry fileEntry, CancellationToken cance
                 var markdown = await ConvertToMarkdownAsync(fileStorageManager, fileEntry, cancellationToken);
                 await File.WriteAllTextAsync(markdownFile, markdown, cancellationToken);
             }
+
+            var chunks = TextChunkingService.ChunkSentences(await File.ReadAllTextAsync(markdownFile, cancellationToken));
+
+            var chunksFolder = Path.Combine(_configuration["Storage:TempFolderPath"], "Chunks", fileEntry.Id.ToString());
+
+            if (!Directory.Exists(chunksFolder))
+            {
+                Directory.CreateDirectory(chunksFolder);
+            }
+
+            foreach (var chunk in chunks)
+            {
+                await File.WriteAllTextAsync(Path.Combine(chunksFolder, $"{chunk.StartIndex}_{chunk.EndIndex}.txt"), chunk.Text, cancellationToken);
+            }
         }
         else if (fileExtension == ".jpg" ||
             fileExtension == ".png")
@@ -234,3 +275,173 @@ class Caption
         public float Confidence { get; set; }
     }
 }
+
+public class Chunk
+{
+    public required string Text { get; init; }
+
+    public required int StartIndex { get; init; }
+
+    public required int EndIndex { get; init; }
+}
+
+public class TextChunkingService
+{
+    public static IEnumerable<Chunk> ChunkSentences(string text, int maxTokens = 800)
+    {
+        // Split text into sentences while preserving their original positions
+        var sentenceMatches = Regex.Matches(text, @"[^\.!\?]*[\.!\?]\s*");
+        var sentences = new List<(string content, int start, int end)>();
+
+        int lastEnd = 0;
+        foreach (Match match in sentenceMatches)
+        {
+            sentences.Add((match.Value, match.Index, match.Index + match.Length - 1));
+            lastEnd = match.Index + match.Length;
+        }
+
+        // Handle any remaining text that doesn't end with sentence punctuation
+        if (lastEnd < text.Length)
+        {
+            string remaining = text.Substring(lastEnd);
+            if (!string.IsNullOrWhiteSpace(remaining))
+            {
+                sentences.Add((remaining, lastEnd, text.Length - 1));
+            }
+        }
+
+        var current = new StringBuilder();
+        int tokenCount = 0;
+        int chunkStartIndex = -1;
+        int chunkEndIndex = -1;
+
+        foreach (var (content, start, end) in sentences)
+        {
+            int sentenceTokens = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Length;
+
+            if (tokenCount + sentenceTokens > maxTokens && current.Length > 0)
+            {
+                yield return new Chunk
+                {
+                    Text = current.ToString().Trim(),
+                    StartIndex = chunkStartIndex,
+                    EndIndex = chunkEndIndex
+                };
+
+                current.Clear();
+                tokenCount = 0;
+                chunkStartIndex = -1;
+            }
+
+            if (current.Length == 0)
+            {
+                chunkStartIndex = start;
+            }
+
+            current.Append(content);
+            tokenCount += sentenceTokens;
+            chunkEndIndex = end;
+        }
+
+        if (current.Length > 0)
+        {
+            yield return new Chunk
+            {
+                Text = current.ToString().Trim(),
+                StartIndex = chunkStartIndex,
+                EndIndex = chunkEndIndex
+            };
+        }
+    }
+
+    public static IEnumerable<Chunk> ChunkSentencesOverlapping(string text, int maxTokens = 800, double overlapRatio = 0.1)
+    {
+        // Split text into sentences while preserving their original positions
+        var sentenceMatches = Regex.Matches(text, @"[^\.!\?]*[\.!\?]\s*");
+        var sentences = new List<(string content, int start, int end)>();
+
+        int lastEnd = 0;
+        foreach (Match match in sentenceMatches)
+        {
+            sentences.Add((match.Value.Trim(), match.Index, match.Index + match.Length - 1));
+            lastEnd = match.Index + match.Length;
+        }
+
+        // Handle remaining text
+        if (lastEnd < text.Length)
+        {
+            string remaining = text.Substring(lastEnd);
+            if (!string.IsNullOrWhiteSpace(remaining))
+            {
+                sentences.Add((remaining.Trim(), lastEnd, text.Length - 1));
+            }
+        }
+
+        var current = new StringBuilder();
+        int tokenCount = 0;
+        int estimatedOverlapTokens = (int)(maxTokens * overlapRatio);
+        var lastChunkSentences = new List<(string content, int start, int end)>();
+        int chunkStartIndex = -1;
+        int chunkEndIndex = -1;
+
+        foreach (var sentence in sentences)
+        {
+            int sentenceTokens = EstimateTokens(sentence.content);
+
+            // If adding this sentence exceeds limit, yield current chunk
+            if (tokenCount + sentenceTokens > maxTokens && current.Length > 0)
+            {
+                yield return new Chunk
+                {
+                    Text = current.ToString().Trim(),
+                    StartIndex = chunkStartIndex,
+                    EndIndex = chunkEndIndex
+                };
+
+                // Prepare overlap from previous sentences
+                var overlapSentences = lastChunkSentences.TakeLast(Math.Max(1, estimatedOverlapTokens / 20)).ToList();
+                if (overlapSentences.Any())
+                {
+                    var overlap = string.Join(" ", overlapSentences.Select(s => s.content));
+                    current.Clear();
+                    current.Append(overlap + " ");
+                    tokenCount = EstimateTokens(overlap);
+                    chunkStartIndex = overlapSentences.First().start;
+                }
+                else
+                {
+                    current.Clear();
+                    tokenCount = 0;
+                    chunkStartIndex = sentence.start;
+                }
+                lastChunkSentences.Clear();
+            }
+
+            if (current.Length == 0)
+            {
+                chunkStartIndex = sentence.start;
+            }
+
+            current.Append(sentence.content + " ");
+            tokenCount += sentenceTokens;
+            chunkEndIndex = sentence.end;
+            lastChunkSentences.Add(sentence);
+        }
+
+        if (current.Length > 0)
+        {
+            yield return new Chunk
+            {
+                Text = current.ToString().Trim(),
+                StartIndex = chunkStartIndex,
+                EndIndex = chunkEndIndex
+            };
+        }
+    }
+
+    private static int EstimateTokens(string text)
+    {
+        // Rough heuristic: 1 token ≈ 4 chars
+        return Math.Max(1, text.Length / 4);
+    }
+}