feat: Redis vector storage for PDF-based RAG

Directory.Packages.props

@@ -43,6 +43,7 @@
     <PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="10.0.7" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.7" />
     <PackageVersion Include="Microsoft.ML.Tokenizers.Data.Cl100kBase" Version="2.0.0" />
+    <PackageVersion Include="PdfPig" Version="0.1.14" />
     <PackageVersion Include="MMALSharp" Version="0.6.0" />
     <PackageVersion Include="MMALSharp.FFmpeg" Version="0.6.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.3.9" />

appsettings.Development.json

@@ -43,10 +43,18 @@
         },
         "EdgeCpuWhisper": {
           "Endpoint": "http://localhost:11434"
+        },
+        "EdgeEmbedding": {
+          "Endpoint": "http://localhost:11434"
         }
       },
       "Agents": {}
     },
+    "RagConfig": {
+      "EmbeddingProvider": "EdgeEmbedding",
+      "DocumentsPath": "./rag-documents",
+      "AutoIngestOnStartup": false
+    },
     "ApiAuthConfig": {
       "Username": "demo",
       "Password": "demo"

appsettings.json

@@ -73,6 +73,11 @@
           "Type": "Ollama",
           "Endpoint": "http://ollama-gpu.utilities.svc:11434",
           "ModelName": "karanchopda333/whisper"
+        },
+        "EdgeEmbedding": {
+          "Type": "Ollama",
+          "Endpoint": "http://ollama-gpu.utilities.svc:11434",
+          "ModelName": "nomic-embed-text"
         }
       },
       "PollTtlMs": 3600000,
@@ -285,6 +290,31 @@
         }
       }
     },
+    "RagConfig": {
+      "EmbeddingProvider": "EdgeEmbedding",
+      "IndexName": "rag-documents",
+      "Dimension": 768,
+      "DistanceMetric": "COSINE",
+      "ChunkSizeTokens": 512,
+      "ChunkOverlapTokens": 50,
+      "TopK": 5,
+      "DocumentsPath": "/data/rag-documents",
+      "AutoIngestOnStartup": true,
+      "AgentSources": {
+        "HeatingAgent": [
+          {
+            "CollectionName": "rag-documents",
+            "TopK": 5
+          }
+        ],
+        "AppliancesAgent": [
+          {
+            "CollectionName": "rag-documents",
+            "TopK": 3
+          }
+        ]
+      }
+    },
     "SignalRHubConfig": {
       "SignalRHub": "http://signalrhub.prd.svc:8080",
       "ConsoleLogIntervalMs": 30000,

docker-compose.yml

@@ -44,8 +44,8 @@ services:
   # ---------- Infrastructure (always started) ----------
 
   redis:
-    image: redis
-    command: [ "redis-server", "/usr/local/etc/redis/redis.conf" ]
+    image: redis/redis-stack-server
+    command: [ "redis-server", "/usr/local/etc/redis/redis.conf", "--loadmodule", "/opt/redis-stack/lib/redisearch.so" ]
     ports:
       - 6379:6379
     volumes:
@@ -121,7 +121,7 @@ services:
         condition: service_healthy
     environment:
       OLLAMA_HOST: http://ollama:11434
-    entrypoint: [ "sh", "-c", "ollama pull qwen3.5:4b" ]
+    entrypoint: [ "sh", "-c", "ollama pull qwen3.5:4b && ollama pull nomic-embed-text" ]
     restart: "no"
 
   # ---------- Demo profile: signal-cli ----------
@@ -188,6 +188,11 @@ services:
       CasCap__AIConfig__Providers__EdgeGpuVL__Endpoint: http://ollama:11434
       CasCap__AIConfig__Providers__EdgeOllamaCpuVLC__Endpoint: http://ollama:11434
       CasCap__AIConfig__Providers__EdgeCpuWhisper__Endpoint: http://ollama:11434
+
+      # RAG — embedding model and document path
+      CasCap__AIConfig__Providers__EdgeEmbedding__Endpoint: http://ollama:11434
+      CasCap__RagConfig__DocumentsPath: /data/rag-documents
+      CasCap__RagConfig__EmbeddingProvider: EdgeEmbedding
     depends_on:
       redis:
         condition: service_healthy

src/CasCap.App.Server/Program.cs

@@ -149,6 +149,13 @@
     if (enabledFeatures.Contains(FeatureNames.Comms))
         builder.AddComms();
 
+    if (enabledFeatures.Contains(FeatureNames.Rag))
+    {
+        builder.AddRag();
+        mcpBuilder.WithToolsFromAssembly(typeof(RagMcpQueryService).Assembly);
+        mcpBuilder.WithPromptsFromAssembly(typeof(RagMcpQueryService).Assembly);
+    }
+
     // Register all AI agent profiles with deferred tool resolution
     var otelSourceName = AgentExtensions.GetAISourceName(appConfig.MetricNamePrefix);
     foreach (var (agentName, agentConfig) in aiConfig.Agents.Where(a => a.Value.Enabled))

src/CasCap.SmartHaus/Abstractions/IDocumentIngestionService.cs

@@ -0,0 +1,34 @@
+namespace CasCap.Abstractions;
+
+/// <summary>
+/// Abstraction for ingesting documents (PDFs) into the vector store.
+/// </summary>
+public interface IDocumentIngestionService
+{
+    /// <summary>
+    /// Ingests a PDF document: extracts text, chunks, generates embeddings, and stores in the vector index.
+    /// </summary>
+    /// <param name="pdfStream">Stream containing the PDF content.</param>
+    /// <param name="documentName">Human-readable document name.</param>
+    /// <param name="collectionName">Target vector collection name.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>Metadata about the ingested document.</returns>
+    Task<DocumentInfo> IngestDocumentAsync(Stream pdfStream, string documentName, string collectionName, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Ingests all PDF files from a directory into the vector store.
+    /// </summary>
+    /// <param name="directoryPath">Absolute path to the directory containing PDF files.</param>
+    /// <param name="collectionName">Target vector collection name.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>Metadata for each ingested document.</returns>
+    Task<List<DocumentInfo>> IngestDirectoryAsync(string directoryPath, string collectionName, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Removes a document and all its chunks from the vector store.
+    /// </summary>
+    /// <param name="documentId">The document identifier to remove.</param>
+    /// <param name="collectionName">Target vector collection name.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task RemoveDocumentAsync(string documentId, string collectionName, CancellationToken cancellationToken = default);
+}

src/CasCap.SmartHaus/Abstractions/IDocumentVectorStore.cs

@@ -0,0 +1,42 @@
+namespace CasCap.Abstractions;
+
+/// <summary>
+/// Abstraction for storing and searching document chunks in a vector database.
+/// </summary>
+public interface IDocumentVectorStore
+{
+    /// <summary>Ensures the vector collection and its index exist in Redis.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task EnsureCollectionAsync(string collectionName, CancellationToken cancellationToken = default);
+
+    /// <summary>Upserts pre-embedded document chunks into the vector store.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="chunks">Chunks with embeddings already populated.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task UpsertChunksAsync(string collectionName, IReadOnlyList<DocumentChunk> chunks, CancellationToken cancellationToken = default);
+
+    /// <summary>Performs a vector similarity search using the provided query embedding.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="queryEmbedding">The query vector.</param>
+    /// <param name="topK">Number of results to return.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    /// <returns>Ranked search results.</returns>
+    Task<List<DocumentSearchResult>> SearchAsync(string collectionName, ReadOnlyMemory<float> queryEmbedding, int topK, CancellationToken cancellationToken = default);
+
+    /// <summary>Removes all chunks belonging to a specific document.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="documentId">The document identifier.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task RemoveDocumentAsync(string collectionName, string documentId, CancellationToken cancellationToken = default);
+
+    /// <summary>Deletes the entire collection and its index.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task DeleteCollectionAsync(string collectionName, CancellationToken cancellationToken = default);
+
+    /// <summary>Returns metadata for all ingested documents in a collection.</summary>
+    /// <param name="collectionName">Logical name of the collection.</param>
+    /// <param name="cancellationToken">Cancellation token.</param>
+    Task<List<DocumentInfo>> ListDocumentsAsync(string collectionName, CancellationToken cancellationToken = default);
+}

src/CasCap.SmartHaus/CasCap.SmartHaus.csproj

@@ -71,6 +71,7 @@
 
   <ItemGroup>
     <PackageReference Include="ModelContextProtocol.AspNetCore" />
+    <PackageReference Include="PdfPig" />
   </ItemGroup>
 
 </Project>

src/CasCap.SmartHaus/Extensions/EmbeddingGeneratorFactory.cs

@@ -0,0 +1,58 @@
+using Microsoft.Extensions.AI;
+using OllamaSharp;
+using System.ClientModel;
+
+namespace CasCap.Extensions;
+
+/// <summary>
+/// Factory methods for creating <see cref="IEmbeddingGenerator{TInput,TEmbedding}"/> instances
+/// from <see cref="ProviderConfig"/>.
+/// </summary>
+/// <remarks>
+/// This is a SmartHaus-local helper that mirrors the <c>AgentExtensions.CreateAgent</c> pattern
+/// from CasCap.Common.AI. When the shared library adds a <c>CreateEmbeddingGenerator</c> method,
+/// this class can be removed in favour of the shared implementation.
+/// </remarks>
+public static class EmbeddingGeneratorFactory
+{
+    /// <summary>
+    /// Creates an <see cref="IEmbeddingGenerator{String, Embedding}"/> from the specified provider configuration.
+    /// </summary>
+    /// <param name="provider">The provider configuration containing endpoint, model, and type.</param>
+    /// <param name="httpClient">Optional pre-configured HTTP client (e.g. with basic auth for dev Ollama).</param>
+    /// <returns>An embedding generator for the configured provider.</returns>
+    public static IEmbeddingGenerator<string, Embedding<float>> CreateEmbeddingGenerator(ProviderConfig provider, HttpClient? httpClient = null) =>
+        provider.Type switch
+        {
+            AgentType.Ollama => CreateOllamaEmbeddingGenerator(provider, httpClient),
+            AgentType.OpenAI => CreateOpenAIEmbeddingGenerator(provider),
+            AgentType.AzureOpenAI => CreateAzureOpenAIEmbeddingGenerator(provider),
+            _ => throw new NotSupportedException($"Embedding generation is not supported for provider type '{provider.Type}'."),
+        };
+
+    private static IEmbeddingGenerator<string, Embedding<float>> CreateOllamaEmbeddingGenerator(ProviderConfig provider, HttpClient? httpClient)
+    {
+        var uri = provider.Endpoint ?? new Uri("http://localhost:11434");
+
+        // OllamaApiClient directly implements IEmbeddingGenerator<string, Embedding<float>>.
+        if (httpClient is not null)
+            return new OllamaApiClient(httpClient, provider.ModelName);
+
+        return new OllamaApiClient(uri, provider.ModelName);
+    }
+
+    private static IEmbeddingGenerator<string, Embedding<float>> CreateOpenAIEmbeddingGenerator(ProviderConfig provider)
+    {
+        var apiKey = provider.ApiKey ?? throw new InvalidOperationException("OpenAI embedding provider requires an ApiKey.");
+        var client = new OpenAI.OpenAIClient(new ApiKeyCredential(apiKey));
+        return client.GetEmbeddingClient(provider.ModelName).AsIEmbeddingGenerator();
+    }
+
+    private static IEmbeddingGenerator<string, Embedding<float>> CreateAzureOpenAIEmbeddingGenerator(ProviderConfig provider)
+    {
+        var endpoint = provider.Endpoint ?? throw new InvalidOperationException("AzureOpenAI embedding provider requires an Endpoint.");
+        var apiKey = provider.ApiKey ?? throw new InvalidOperationException("AzureOpenAI embedding provider requires an ApiKey.");
+        var client = new Azure.AI.OpenAI.AzureOpenAIClient(endpoint, new AzureKeyCredential(apiKey));
+        return client.GetEmbeddingClient(provider.ModelName).AsIEmbeddingGenerator();
+    }
+}

src/CasCap.SmartHaus/Extensions/McpServiceCollectionExtensions.cs

@@ -1,3 +1,6 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.Extensions.AI;
+
 namespace CasCap.Extensions;
 
 /// <summary>
@@ -104,4 +107,45 @@ public static void AddMessagingMcpStub(this IServiceCollection services)
             string.Empty,
             string.Empty));
     }
+
+    /// <summary>
+    /// Registers <see cref="Services.RagMcpQueryService"/> and its dependencies
+    /// (<see cref="IDocumentVectorStore"/>, <see cref="IDocumentIngestionService"/>,
+    /// embedding generator, and configuration) for RAG document management tools.
+    /// </summary>
+    /// <param name="builder">The web application builder.</param>
+    public static void AddRag(this WebApplicationBuilder builder)
+    {
+        builder.Services.AddCasCapConfiguration<RagConfig>();
+        builder.Services.AddSingleton<IDocumentVectorStore, RedisDocumentVectorStore>();
+        builder.Services.AddSingleton<IDocumentIngestionService, PdfDocumentIngestionService>();
+
+        // Register IEmbeddingGenerator from the configured provider.
+        builder.Services.AddSingleton(sp =>
+        {
+            var ragCfg = sp.GetRequiredService<IOptions<RagConfig>>().Value;
+            var aiCfg = sp.GetRequiredService<IOptions<AIConfig>>().Value;
+
+            if (!aiCfg.Providers.TryGetValue(ragCfg.EmbeddingProvider, out var providerConfig))
+                throw new InvalidOperationException(
+                    $"Embedding provider '{ragCfg.EmbeddingProvider}' not found in AIConfig.Providers.");
+
+            HttpClient? httpClient = null;
+            if (providerConfig.Type is AgentType.Ollama && builder.Environment.IsDevelopment())
+            {
+                httpClient = new HttpClient
+                {
+                    BaseAddress = providerConfig.Endpoint,
+                    Timeout = Timeout.InfiniteTimeSpan,
+                };
+                var authOpts = sp.GetRequiredService<IOptions<ApiAuthConfig>>().Value;
+                httpClient.SetBasicAuth(authOpts.Username, authOpts.Password);
+            }
+
+            return EmbeddingGeneratorFactory.CreateEmbeddingGenerator(providerConfig, httpClient);
+        });
+
+        builder.Services.AddSingleton<RagMcpQueryService>();
+        builder.Services.AddSingleton<IBgFeature, RagIngestionBgService>();
+    }
 }

src/CasCap.SmartHaus/Models/DocumentChunk.cs

@@ -0,0 +1,29 @@
+namespace CasCap.Models;
+
+/// <summary>
+/// A chunked section of an ingested document with its embedding vector, stored in Redis
+/// as a hash with a vector field for similarity search.
+/// </summary>
+public class DocumentChunk
+{
+    /// <summary>Unique chunk identifier (<c>{documentId}:{chunkIndex}</c>).</summary>
+    public required string Id { get; init; }
+
+    /// <summary>Stable identifier for the source document (derived from the file name).</summary>
+    public required string DocumentId { get; init; }
+
+    /// <summary>Human-readable document name (e.g. the original PDF filename).</summary>
+    public required string DocumentName { get; init; }
+
+    /// <summary>Plain-text content of this chunk.</summary>
+    public required string Content { get; init; }
+
+    /// <summary>One-based page number in the source PDF where this chunk starts.</summary>
+    public int PageNumber { get; init; }
+
+    /// <summary>Zero-based index of this chunk within its parent document.</summary>
+    public int ChunkIndex { get; init; }
+
+    /// <summary>Embedding vector generated from <see cref="Content"/>.</summary>
+    public ReadOnlyMemory<float> Embedding { get; set; }
+}

src/CasCap.SmartHaus/Models/DocumentInfo.cs

@@ -0,0 +1,27 @@
+namespace CasCap.Models;
+
+/// <summary>
+/// Metadata about an ingested document.
+/// </summary>
+public record DocumentInfo
+{
+    /// <summary>Stable identifier for the document (derived from the file name).</summary>
+    [Description("Unique document identifier.")]
+    public required string DocumentId { get; init; }
+
+    /// <summary>Original file name of the document.</summary>
+    [Description("Original PDF file name.")]
+    public required string DocumentName { get; init; }
+
+    /// <summary>Number of pages extracted from the PDF.</summary>
+    [Description("Total pages in the source PDF.")]
+    public int PageCount { get; init; }
+
+    /// <summary>Number of chunks generated from the document.</summary>
+    [Description("Number of text chunks stored in the vector index.")]
+    public int ChunkCount { get; init; }
+
+    /// <summary>UTC timestamp when the document was ingested.</summary>
+    [Description("When the document was last ingested (UTC).")]
+    public DateTime IngestedAtUtc { get; init; }
+}

src/CasCap.SmartHaus/Models/DocumentSearchResult.cs

@@ -0,0 +1,23 @@
+namespace CasCap.Models;
+
+/// <summary>
+/// A single result from a vector similarity search against ingested documents.
+/// </summary>
+public record DocumentSearchResult
+{
+    /// <summary>Human-readable document name.</summary>
+    [Description("Source document name.")]
+    public required string DocumentName { get; init; }
+
+    /// <summary>One-based page number in the source PDF.</summary>
+    [Description("Page number in the source PDF.")]
+    public int PageNumber { get; init; }
+
+    /// <summary>Similarity score (lower is more similar for cosine distance).</summary>
+    [Description("Similarity score — lower values indicate higher relevance.")]
+    public double Score { get; init; }
+
+    /// <summary>The text content of the matched chunk.</summary>
+    [Description("Matched text content from the document.")]
+    public required string Content { get; init; }
+}