Add AI-enriched fields

src/Elastic.Documentation.Configuration/DocumentationEndpoints.cs

@@ -45,4 +45,10 @@ public class ElasticsearchEndpoint
 	public int? BootstrapTimeout { get; set; }
 	public bool NoSemantic { get; set; }
 	public bool ForceReindex { get; set; }
+
+	/// <summary>
+	/// Enable AI enrichment of documents using LLM-generated metadata.
+	/// When enabled, documents are enriched with summaries, search queries, and questions.
+	/// </summary>
+	public bool EnableAiEnrichment { get; set; }
 }

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -83,4 +83,41 @@ public record DocumentationDocument
 	[JsonPropertyName("hidden")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
 	public bool Hidden { get; set; }
+
+	// AI Enrichment fields - populated by DocumentEnrichmentService
+
+	/// <summary>
+	/// 3-5 sentences dense with technical entities, API names, and core functionality for vector matching.
+	/// </summary>
+	[JsonPropertyName("ai_rag_optimized_summary")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public string? AiRagOptimizedSummary { get; set; }
+
+	/// <summary>
+	/// Exactly 5-10 words for a UI tooltip.
+	/// </summary>
+	[JsonPropertyName("ai_short_summary")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public string? AiShortSummary { get; set; }
+
+	/// <summary>
+	/// A 3-8 word keyword string representing a high-intent user search for this doc.
+	/// </summary>
+	[JsonPropertyName("ai_search_query")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public string? AiSearchQuery { get; set; }
+
+	/// <summary>
+	/// Array of 3-5 specific questions answered by this document.
+	/// </summary>
+	[JsonPropertyName("ai_questions")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public string[]? AiQuestions { get; set; }
+
+	/// <summary>
+	/// Array of 2-4 specific use cases this doc helps with.
+	/// </summary>
+	[JsonPropertyName("ai_use_cases")]
+	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+	public string[]? AiUseCases { get; set; }
 }

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchIngestChannel.Mapping.cs

@@ -201,6 +201,32 @@ protected static string CreateMapping(string? inferenceId) =>
 		        "fields" : {
 		          {{(!string.IsNullOrWhiteSpace(inferenceId) ? $"\"semantic_text\": {{{InferenceMapping(inferenceId)}}}" : "")}}
 		        }
+		      },
+		      "ai_rag_optimized_summary": {
+		        "type": "text",
+		        "analyzer": "synonyms_fixed_analyzer",
+		        "search_analyzer": "synonyms_analyzer",
+		        "fields": {
+		          {{(!string.IsNullOrWhiteSpace(inferenceId) ? $"\"semantic_text\": {{{InferenceMapping(inferenceId)}}}" : "")}}
+		        }
+		      },
+		      "ai_short_summary": {
+		        "type": "text"
+		      },
+		      "ai_search_query": {
+		        "type": "keyword"
+		      },
+		      "ai_questions": {
+		        "type": "text",
+		        "fields": {
+		          {{(!string.IsNullOrWhiteSpace(inferenceId) ? $"\"semantic_text\": {{{InferenceMapping(inferenceId)}}}" : "")}}
+		        }
+		      },
+		      "ai_use_cases": {
+		        "type": "text",
+		        "fields": {
+		          {{(!string.IsNullOrWhiteSpace(inferenceId) ? $"\"semantic_text\": {{{InferenceMapping(inferenceId)}}}" : "")}}
+		        }
 		      }
 		    }
 		  }

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.Export.cs

@@ -131,6 +131,10 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 		};
 
 		CommonEnrichments(doc, currentNavigation);
+
+		// AI enrichment - respects per-run limit, uses cache
+		_ = await _enrichmentService.TryEnrichAsync(doc, ctx);
+
 		AssignDocumentMetadata(doc);
 
 		if (_indexStrategy == IngestStrategy.Multiplex)
@@ -166,6 +170,10 @@ public async ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Canc
 			doc.Abstract = @abstract;
 			doc.Headings = headings;
 			CommonEnrichments(doc, null);
+
+			// AI enrichment - respects per-run limit, uses cache
+			_ = await _enrichmentService.TryEnrichAsync(doc, ctx);
+
 			AssignDocumentMetadata(doc);
 
 			// Write to channels following the multiplex or reindex strategy

src/Elastic.Markdown/Exporters/Elasticsearch/ElasticsearchMarkdownExporter.cs

@@ -10,6 +10,7 @@
 using Elastic.Documentation.Diagnostics;
 using Elastic.Ingest.Elasticsearch;
 using Elastic.Ingest.Elasticsearch.Indices;
+using Elastic.Markdown.Exporters.Elasticsearch.Enrichment;
 using Elastic.Transport;
 using Elastic.Transport.Products.Elasticsearch;
 using Microsoft.Extensions.Logging;
@@ -41,6 +42,9 @@ public partial class ElasticsearchMarkdownExporter : IMarkdownExporter, IDisposa
 	private readonly IReadOnlyCollection<QueryRule> _rules;
 	private readonly VersionsConfiguration _versionsConfiguration;
 	private readonly string _fixedSynonymsHash;
+	private readonly Enrichment.DocumentEnrichmentService _enrichmentService;
+	private readonly IEnrichmentCache _enrichmentCache;
+	private readonly ILlmClient _llmClient;
 
 	public ElasticsearchMarkdownExporter(
 		ILoggerFactory logFactory,
@@ -97,6 +101,16 @@ IDocumentationConfigurationContext context
 
 		_lexicalChannel = new ElasticsearchLexicalIngestChannel(logFactory, collector, es, indexNamespace, _transport, indexTimeSynonyms);
 		_semanticChannel = new ElasticsearchSemanticIngestChannel(logFactory, collector, es, indexNamespace, _transport, indexTimeSynonyms);
+
+		// Create enrichment services
+		var enrichmentOptions = new EnrichmentOptions { Enabled = es.EnableAiEnrichment };
+		_enrichmentCache = new ElasticsearchEnrichmentCache(_transport, logFactory.CreateLogger<ElasticsearchEnrichmentCache>());
+		_llmClient = new ElasticsearchLlmClient(_transport, logFactory.CreateLogger<ElasticsearchLlmClient>());
+		_enrichmentService = new Enrichment.DocumentEnrichmentService(
+			_enrichmentCache,
+			_llmClient,
+			enrichmentOptions,
+			logFactory.CreateLogger<Enrichment.DocumentEnrichmentService>());
 	}
 
 	/// <inheritdoc />
@@ -105,6 +119,7 @@ public async ValueTask StartAsync(Cancel ctx = default)
 		_currentLexicalHash = await _lexicalChannel.Channel.GetIndexTemplateHashAsync(ctx) ?? string.Empty;
 		_currentSemanticHash = await _semanticChannel.Channel.GetIndexTemplateHashAsync(ctx) ?? string.Empty;
 
+		await _enrichmentService.InitializeAsync(ctx);
 		await PublishSynonymsAsync(ctx);
 		await PublishQueryRulesAsync(ctx);
 		_ = await _lexicalChannel.Channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, null, ctx);
@@ -230,6 +245,9 @@ private async ValueTask<long> CountAsync(string index, string body, Cancel ctx =
 	/// <inheritdoc />
 	public async ValueTask StopAsync(Cancel ctx = default)
 	{
+		// Log AI enrichment progress
+		_enrichmentService.LogProgress();
+
 		var semanticWriteAlias = string.Format(_semanticChannel.Channel.Options.IndexFormat, "latest");
 		var lexicalWriteAlias = string.Format(_lexicalChannel.Channel.Options.IndexFormat, "latest");
 
@@ -436,6 +454,8 @@ public void Dispose()
 	{
 		_lexicalChannel.Dispose();
 		_semanticChannel.Dispose();
+		_enrichmentService.Dispose();
+		_llmClient.Dispose();
 		GC.SuppressFinalize(this);
 	}
 }

src/Elastic.Markdown/Exporters/Elasticsearch/Enrichment/DocumentEnrichmentService.cs

@@ -0,0 +1,225 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;
+using Elastic.Documentation.Search;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Markdown.Exporters.Elasticsearch.Enrichment;
+
+/// <summary>
+/// Orchestrates document enrichment using an LLM client and cache.
+/// </summary>
+public sealed partial class DocumentEnrichmentService(
+	IEnrichmentCache cache,
+	ILlmClient llm,
+	EnrichmentOptions options,
+	ILogger<DocumentEnrichmentService> logger) : IDisposable
+{
+	private readonly IEnrichmentCache _cache = cache;
+	private readonly ILlmClient _llm = llm;
+	private readonly EnrichmentOptions _options = options;
+	private readonly ILogger _logger = logger;
+
+	private int _cacheHitCount;
+	private int _staleRefreshCount;
+	private int _newEnrichmentCount;
+	private int _skippedCount;
+
+	public Task InitializeAsync(CancellationToken ct) =>
+		_options.Enabled ? _cache.InitializeAsync(ct) : Task.CompletedTask;
+
+	public async Task<bool> TryEnrichAsync(DocumentationDocument doc, CancellationToken ct)
+	{
+		if (!_options.Enabled)
+			return false;
+
+		if (string.IsNullOrWhiteSpace(doc.StrippedBody))
+			return false;
+
+		var cacheKey = GenerateCacheKey(doc.Title, doc.StrippedBody);
+
+		if (TryApplyCachedEnrichment(doc, cacheKey))
+		{
+			await TryRefreshStaleCacheAsync(doc, cacheKey, ct);
+			return true;
+		}
+
+		return await TryEnrichNewDocumentAsync(doc, cacheKey, ct);
+	}
+
+	public void LogProgress()
+	{
+		if (!_options.Enabled)
+		{
+			_logger.LogInformation("AI enrichment is disabled (use --enable-ai-enrichment to enable)");
+			return;
+		}
+
+		_logger.LogInformation(
+			"Enrichment summary: {CacheHits} cache hits ({StaleRefreshed} stale refreshed), {NewEnrichments} new, {Skipped} skipped (limit: {Limit})",
+			_cacheHitCount, _staleRefreshCount, _newEnrichmentCount, _skippedCount, _options.MaxNewEnrichmentsPerRun);
+
+		if (_skippedCount > 0)
+		{
+			_logger.LogInformation(
+				"Enrichment progress: {Skipped} documents pending, will complete over subsequent runs",
+				_skippedCount);
+		}
+	}
+
+	public void Dispose() => (_llm as IDisposable)?.Dispose();
+
+	private bool TryApplyCachedEnrichment(DocumentationDocument doc, string cacheKey)
+	{
+		var cached = _cache.TryGet(cacheKey);
+		if (cached is null)
+			return false;
+
+		// Defensive check: if cached data is invalid, treat as miss and let it re-enrich
+		if (!cached.Data.HasData)
+		{
+			_logger.LogDebug("Cached entry for {Url} has no valid data, will re-enrich", doc.Url);
+			return false;
+		}
+
+		ApplyEnrichment(doc, cached.Data);
+		_ = Interlocked.Increment(ref _cacheHitCount);
+		return true;
+	}
+
+	private async Task TryRefreshStaleCacheAsync(DocumentationDocument doc, string cacheKey, CancellationToken ct)
+	{
+		var cached = _cache.TryGet(cacheKey);
+		// If cache is current version or newer, no refresh needed
+		if (cached is not null && cached.PromptVersion >= _options.PromptVersion)
+			return;
+
+		if (!TryClaimEnrichmentSlot())
+			return;
+
+		_ = Interlocked.Increment(ref _staleRefreshCount);
+
+		try
+		{
+			var fresh = await _llm.EnrichAsync(doc.Title, doc.StrippedBody ?? string.Empty, ct);
+			if (fresh is not null)
+			{
+				await _cache.StoreAsync(cacheKey, fresh, _options.PromptVersion, ct);
+				ApplyEnrichment(doc, fresh);
+			}
+		}
+		catch (Exception ex) when (ex is not OperationCanceledException)
+		{
+			_logger.LogDebug(ex, "Failed to refresh stale cache for {Url}", doc.Url);
+		}
+	}
+
+	private async Task<bool> TryEnrichNewDocumentAsync(DocumentationDocument doc, string cacheKey, CancellationToken ct)
+	{
+		if (!TryClaimEnrichmentSlot())
+		{
+			_logger.LogDebug("Skipping enrichment for {Url} - limit reached", doc.Url);
+			_ = Interlocked.Increment(ref _skippedCount);
+			return false;
+		}
+
+		try
+		{
+			var enrichment = await _llm.EnrichAsync(doc.Title, doc.StrippedBody ?? string.Empty, ct);
+			if (enrichment is not null)
+			{
+				await _cache.StoreAsync(cacheKey, enrichment, _options.PromptVersion, ct);
+				ApplyEnrichment(doc, enrichment);
+				return true;
+			}
+		}
+		catch (Exception ex) when (ex is not OperationCanceledException)
+		{
+			_logger.LogWarning(ex, "Failed to enrich document {Url}", doc.Url);
+		}
+
+		return false;
+	}
+
+	/// <summary>
+	/// Tries to get permission to make a new LLM call.
+	/// 
+	/// Why: We have ~12k documents but only allow 100 new LLM calls per deployment.
+	/// This keeps each deployment fast. Documents not enriched this run will be
+	/// enriched in the next deployment. Cache hits are free and don't count.
+	/// 
+	/// How: Add 1 to counter. If counter is too high, subtract 1 and return false.
+	/// This is safe when multiple documents run at the same time.
+	/// </summary>
+	private bool TryClaimEnrichmentSlot()
+	{
+		var current = Interlocked.Increment(ref _newEnrichmentCount);
+		if (current <= _options.MaxNewEnrichmentsPerRun)
+			return true;
+
+		_ = Interlocked.Decrement(ref _newEnrichmentCount);
+		return false;
+	}
+
+	private static void ApplyEnrichment(DocumentationDocument doc, EnrichmentData data)
+	{
+		doc.AiRagOptimizedSummary = data.RagOptimizedSummary;
+		doc.AiShortSummary = data.ShortSummary;
+		doc.AiSearchQuery = data.SearchQuery;
+		doc.AiQuestions = data.Questions;
+		doc.AiUseCases = data.UseCases;
+	}
+
+	private static string GenerateCacheKey(string title, string body)
+	{
+		var normalized = NormalizeContent(title + body);
+		var hash = SHA256.HashData(Encoding.UTF8.GetBytes(normalized));
+		return Convert.ToHexString(hash).ToLowerInvariant();
+	}
+
+	private static string NormalizeContent(string input) =>
+		NormalizeRegex().Replace(input, "").ToLowerInvariant();
+
+	[GeneratedRegex("[^a-zA-Z0-9]")]
+	private static partial Regex NormalizeRegex();
+}
+
+/// <summary>
+/// LLM-generated enrichment data for documentation documents.
+/// </summary>
+public sealed record EnrichmentData
+{
+	[JsonPropertyName("ai_rag_optimized_summary")]
+	public string? RagOptimizedSummary { get; init; }
+
+	[JsonPropertyName("ai_short_summary")]
+	public string? ShortSummary { get; init; }
+
+	[JsonPropertyName("ai_search_query")]
+	public string? SearchQuery { get; init; }
+
+	[JsonPropertyName("ai_questions")]
+	public string[]? Questions { get; init; }
+
+	[JsonPropertyName("ai_use_cases")]
+	public string[]? UseCases { get; init; }
+
+	[JsonIgnore]
+	public bool HasData =>
+		!string.IsNullOrEmpty(RagOptimizedSummary) ||
+		!string.IsNullOrEmpty(ShortSummary) ||
+		!string.IsNullOrEmpty(SearchQuery) ||
+		Questions is { Length: > 0 } ||
+		UseCases is { Length: > 0 };
+}
+
+[JsonSerializable(typeof(EnrichmentData))]
+[JsonSerializable(typeof(CachedEnrichment))]
+[JsonSerializable(typeof(CompletionResponse))]
+[JsonSerializable(typeof(InferenceRequest))]
+internal sealed partial class EnrichmentSerializerContext : JsonSerializerContext;