(#270) Convert documents to Markdown

Author: phongnguyend

src/Monolith/ClassifiedAds.Background/MessageBusConsumers/FileEmbeddingConsumer.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using System;
+using System.IO;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Security.Cryptography;
@@ -40,72 +41,68 @@ public async Task HandleAsync(FileCreatedEvent data, MetaData metaData, Cancella
     {
         _logger.LogInformation("Handling FileCreatedEvent for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
 
-        if (string.IsNullOrEmpty(data?.FileEntry?.FileLocation))
-        {
-            return;
-        }
-
-        if (data.FileEntry.FileName.EndsWith(".txt") ||
-           data.FileEntry.FileName.EndsWith(".md") ||
-           data.FileEntry.FileName.EndsWith(".markdown"))
-        {
-            _logger.LogInformation("Skipping text file for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
-            return;
-        }
+        await ProcessFileAsync(data.FileEntry, cancellationToken);
+    }
 
-        if (data.FileEntry.FileName.EndsWith(".pdf") ||
-            data.FileEntry.FileName.EndsWith(".docx"))
-        {
-            _logger.LogInformation("Converting file to markdown for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
+    public async Task HandleAsync(FileUpdatedEvent data, MetaData metaData, CancellationToken cancellationToken = default)
+    {
+        _logger.LogInformation("Handling FileUpdatedEvent for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
 
-            var markdown = await ConvertToMarkdownAsync(data.FileEntry, cancellationToken);
+        await ProcessFileAsync(data.FileEntry, cancellationToken);
+    }
 
-            return;
-        }
+    public Task HandleAsync(FileDeletedEvent data, MetaData metaData, CancellationToken cancellationToken = default)
+    {
+        _logger.LogInformation("Handling FileDeletedEvent for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
 
-        return;
+        return Task.CompletedTask;
     }
 
-    public async Task HandleAsync(FileUpdatedEvent data, MetaData metaData, CancellationToken cancellationToken = default)
+    private async Task ProcessFileAsync(FileEntry fileEntry, CancellationToken cancellationToken)
     {
-        _logger.LogInformation("Handling FileUpdatedEvent for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
-
-        if (string.IsNullOrEmpty(data?.FileEntry?.FileLocation))
+        if (string.IsNullOrEmpty(fileEntry?.FileLocation))
         {
             return;
         }
 
-        if (data.FileEntry.FileName.EndsWith(".txt") ||
-           data.FileEntry.FileName.EndsWith(".md") ||
-           data.FileEntry.FileName.EndsWith(".markdown"))
+        using var scope = _serviceProvider.CreateScope();
+        var fileStorageManager = scope.ServiceProvider.GetService<IFileStorageManager>();
+
+        var fileExtension = Path.GetExtension(fileEntry.FileName);
+
+        if (fileExtension == ".txt" ||
+           fileExtension == ".md" ||
+           fileExtension == ".markdown")
         {
-            _logger.LogInformation("Skipping text file for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
+            // TODO: xxx
             return;
         }
 
-        if (data.FileEntry.FileName.EndsWith(".pdf") ||
-            data.FileEntry.FileName.EndsWith(".docx"))
+        if (fileExtension == ".pdf" ||
+            fileExtension == ".docx")
         {
-            _logger.LogInformation("Converting file to markdown for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
+            _logger.LogInformation("Converting file to markdown for FileEntry Id: {FileEntryId}", fileEntry?.Id);
 
-            var markdown = await ConvertToMarkdownAsync(data.FileEntry, cancellationToken);
+            var markdownFolder = Path.Combine(_configuration["Storage:TempFolderPath"], "Markdown");
 
-            return;
-        }
+            if (!Directory.Exists(markdownFolder))
+            {
+                Directory.CreateDirectory(markdownFolder);
+            }
 
-        return;
-    }
+            var markdownFile = Path.Combine(markdownFolder, fileEntry.Id + ".md");
 
-    public Task HandleAsync(FileDeletedEvent data, MetaData metaData, CancellationToken cancellationToken = default)
-    {
-        _logger.LogInformation("Handling FileDeletedEvent for FileEntry Id: {FileEntryId}", data?.FileEntry?.Id);
-        return Task.CompletedTask;
+            if (!File.Exists(markdownFile))
+            {
+                var markdown = await ConvertToMarkdownAsync(fileStorageManager, fileEntry, cancellationToken);
+                await File.WriteAllTextAsync(markdownFile, markdown, cancellationToken);
+            }
+        }
     }
 
-    private async Task<string> ConvertToMarkdownAsync(FileEntry fileEntry, CancellationToken cancellationToken = default)
+    private async Task<string> ConvertToMarkdownAsync(IFileStorageManager fileStorageManager, FileEntry fileEntry, CancellationToken cancellationToken = default)
     {
-        // TODO: xxx
-        var content = await _serviceProvider.CreateScope().ServiceProvider.GetRequiredService<IFileStorageManager>().ReadAsync(fileEntry, cancellationToken);
+        var content = await fileStorageManager.ReadAsync(fileEntry, cancellationToken);
 
         if (fileEntry.Encrypted)
         {

src/Monolith/ClassifiedAds.Background/appsettings.json

@@ -30,6 +30,7 @@
   "Storage": {
     "Provider": "Local",
     "MasterEncryptionKey": "+2ZC9wrwlvPswPxCND0BjrKJ3CfOpImGtn4hloVwo2I=",
+    "TempFolderPath": "C:\\Data\\Practical.CleanArchitecture\\Temp",
     "Local": {
       "Path": "C:\\Data\\files"
     },

src/Monolith/ClassifiedAds.Infrastructure/Storages/StorageOptions.cs

@@ -10,6 +10,8 @@ public class StorageOptions
 
     public string MasterEncryptionKey { get; set; }
 
+    public string TempFolderPath { get; set; }
+
     public LocalOptions Local { get; set; }
 
     public AzureBlobOption Azure { get; set; }