WIP - Needs review and cleanup before being merged into main branch

Author: niemyjski

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.5
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.1
     environment:
       discovery.type: single-node
       xpack.security.enabled: 'false'
@@ -19,7 +19,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:8.19.5
+    image: docker.elastic.co/kibana/kibana:9.2.1
     ports:
       - 5601:5601
     networks:

src/Foundatio.Parsers.ElasticQueries/AggregationMap.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using Elastic.Clients.Elasticsearch.Aggregations;
 
 namespace Foundatio.Parsers.ElasticQueries;
 
@@ -8,4 +9,93 @@ public record AggregationMap(string Name, object Value)
     public object Value { get; set; } = Value;
     public List<AggregationMap> Aggregations { get; } = new();
     public Dictionary<string, object> Meta { get; } = new();
+
+    public IDictionary<string, Aggregation> ToDictionary()
+    {
+        var result = new Dictionary<string, Aggregation>();
+
+        // If this is a root container (Value is null), process all sub-aggregations
+        if (Value == null)
+        {
+            foreach (var subAgg in Aggregations)
+            {
+                var subAggregation = CreateAggregation(subAgg);
+                if (subAggregation != null)
+                    result[subAgg.Name] = subAggregation;
+            }
+        }
+        else
+        {
+            AddToDictionary(this, result);
+        }
+
+        return result;
+    }
+
+    private static void AddToDictionary(AggregationMap map, Dictionary<string, Aggregation> result)
+    {
+        if (map?.Value == null)
+            return;
+
+        var aggregation = CreateAggregation(map);
+        if (aggregation != null)
+            result[map.Name] = aggregation;
+    }
+
+    private static Aggregation CreateAggregation(AggregationMap map)
+    {
+        if (map?.Value == null)
+            return null;
+
+        var aggregation = map.Value switch
+        {
+            TermsAggregation terms => new Aggregation { Terms = terms },
+            DateHistogramAggregation dateHistogram => new Aggregation { DateHistogram = dateHistogram },
+            HistogramAggregation histogram => new Aggregation { Histogram = histogram },
+            MinAggregation min => new Aggregation { Min = min },
+            MaxAggregation max => new Aggregation { Max = max },
+            AverageAggregation avg => new Aggregation { Avg = avg },
+            SumAggregation sum => new Aggregation { Sum = sum },
+            StatsAggregation stats => new Aggregation { Stats = stats },
+            ExtendedStatsAggregation extendedStats => new Aggregation { ExtendedStats = extendedStats },
+            CardinalityAggregation cardinality => new Aggregation { Cardinality = cardinality },
+            MissingAggregation missing => new Aggregation { Missing = missing },
+            TopHitsAggregation topHits => new Aggregation { TopHits = topHits },
+            PercentilesAggregation percentiles => new Aggregation { Percentiles = percentiles },
+            GeohashGridAggregation geohashGrid => new Aggregation { GeohashGrid = geohashGrid },
+            _ => null
+        };
+
+        if (aggregation == null)
+            return null;
+
+        // Add sub-aggregations
+        if (map.Aggregations.Count > 0)
+        {
+            aggregation.Aggregations = new Dictionary<string, Aggregation>();
+            foreach (var subAgg in map.Aggregations)
+            {
+                var subAggregation = CreateAggregation(subAgg);
+                if (subAggregation != null)
+                    aggregation.Aggregations[subAgg.Name] = subAggregation;
+            }
+        }
+
+        // Add meta (exclude null values)
+        if (map.Meta.Count > 0)
+        {
+            aggregation.Meta = new Dictionary<string, object>();
+            foreach (var kvp in map.Meta)
+            {
+                if (kvp.Value != null)
+                    aggregation.Meta[kvp.Key] = kvp.Value;
+            }
+
+            // Remove meta if empty after filtering nulls
+            if (aggregation.Meta.Count == 0)
+                aggregation.Meta = null;
+        }
+
+        return aggregation;
+    }
 }

src/Foundatio.Parsers.ElasticQueries/ElasticMappingResolver.cs

@@ -2,6 +2,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using Elastic.Clients.Elasticsearch;
 using Elastic.Clients.Elasticsearch.IndexManagement;
 using Elastic.Clients.Elasticsearch.Mapping;
@@ -18,6 +19,7 @@ public class ElasticMappingResolver
     private readonly TypeMapping _codeMapping;
     private readonly Inferrer _inferrer;
     private readonly ConcurrentDictionary<string, FieldMapping> _mappingCache = new();
+    private readonly ConditionalWeakTable<IProperty, ConcurrentDictionary<string, object>> _propertyMetadata = new();
     private readonly ILogger _logger;
 
     public static ElasticMappingResolver NullInstance = new(() => null);
@@ -85,15 +87,28 @@ public FieldMapping GetMapping(string field, bool followAlias = false)
         {
             string fieldPart = fieldParts[depth];
             IProperty fieldMapping = null;
+            PropertyName foundPropertyName = null;
             if (currentProperties == null || !currentProperties.TryGetProperty(fieldPart, out fieldMapping))
             {
-                // check to see if there is a name match
+                // check to see if there is a name match by iterating through the dictionary keys
                 if (currentProperties != null)
-                    fieldMapping = ((IDictionary<PropertyName, IProperty>)currentProperties).Values.FirstOrDefault(m =>
+                {
+                    foreach (var kvp in (IDictionary<PropertyName, IProperty>)currentProperties)
                     {
-                        string propertyName = _inferrer.PropertyName(m?.TryGetName());
-                        return propertyName != null && propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase);
-                    });
+                        string propertyName = null;
+                        if (_inferrer != null && kvp.Key?.Name != null)
+                            propertyName = _inferrer.PropertyName(kvp.Key);
+                        else if (kvp.Key?.Name != null)
+                            propertyName = kvp.Key.Name;
+
+                        if (propertyName != null && propertyName.Equals(fieldPart, StringComparison.OrdinalIgnoreCase))
+                        {
+                            fieldMapping = kvp.Value;
+                            foundPropertyName = kvp.Key;
+                            break;
+                        }
+                    }
+                }
 
                 // no mapping found, call GetServerMapping again in case it hasn't been called recently and there are possibly new mappings
                 if (fieldMapping == null && GetServerMapping())
@@ -122,17 +137,32 @@ public FieldMapping GetMapping(string field, bool followAlias = false)
                     break;
                 }
             }
+            else
+            {
+                // TryGetProperty succeeded, store the PropertyName used
+                foreach (var kvp in (IDictionary<PropertyName, IProperty>)currentProperties)
+                {
+                    if (kvp.Value == fieldMapping)
+                    {
+                        foundPropertyName = kvp.Key;
+                        break;
+                    }
+                }
+            }
 
-            // coded properties sometimes have null Name properties
-            string name = fieldMapping.TryGetName();
-            // TODO: ?
-            // if (name == null && fieldMapping is IPropertyWithClrOrigin clrOrigin && clrOrigin.ClrOrigin != null)
-            //     name = new PropertyName(clrOrigin.ClrOrigin);
+            // Determine the property name - use foundPropertyName if available, otherwise fall back to fieldPart
+            string resolvedName;
+            if (foundPropertyName != null && _inferrer != null && foundPropertyName.Name != null)
+                resolvedName = _inferrer.PropertyName(foundPropertyName);
+            else if (foundPropertyName != null && foundPropertyName.Name != null)
+                resolvedName = foundPropertyName.Name;
+            else
+                resolvedName = fieldPart;
 
             if (depth == 0)
-                resolvedFieldName += _inferrer.PropertyName(name);
+                resolvedFieldName += resolvedName;
             else
-                resolvedFieldName += "." + _inferrer.PropertyName(name);
+                resolvedFieldName += "." + resolvedName;
 
             if (depth == fieldParts.Length - 1)
             {
@@ -150,6 +180,10 @@ public FieldMapping GetMapping(string field, bool followAlias = false)
             {
                 currentProperties = objectProperty.Properties;
             }
+            else if (fieldMapping is NestedProperty nestedProperty)
+            {
+                currentProperties = nestedProperty.Properties;
+            }
             else
             {
                 if (fieldMapping is TextProperty textProperty)
@@ -415,12 +449,13 @@ private Properties MergeProperties(Properties codeProperties, Properties serverP
                     if (kvp.Value is not FieldAliasProperty aliasProperty)
                         continue;
 
-                    mergedCodeProperties[kvp.Key] = new FieldAliasProperty
+                    var newAliasProperty = new FieldAliasProperty
                     {
-                        //LocalMetadata = aliasProperty.LocalMetadata,
                         Path = _inferrer?.Field(aliasProperty.Path) ?? aliasProperty.Path,
                         // Name = aliasProperty.Name
                     };
+                    CopyPropertyMetadata(aliasProperty, newAliasProperty);
+                    mergedCodeProperties[kvp.Key] = newAliasProperty;
                 }
             }
         }
@@ -433,11 +468,12 @@ private Properties MergeProperties(Properties codeProperties, Properties serverP
         foreach (var serverProperty in serverProperties)
         {
             var merged = serverProperty.Value;
-            // if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty))
-            //     merged.LocalMetadata = codeProperty.LocalMetadata;
 
             if (mergedCodeProperties.TryGetProperty(serverProperty.Key, out var codeProperty))
             {
+                // Copy local metadata from code property to merged property
+                CopyPropertyMetadata(codeProperty, merged);
+
                 switch (merged)
                 {
                     case ObjectProperty objectProperty:
@@ -491,7 +527,7 @@ private bool GetServerMapping()
         }
     }
 
-    public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, TypeMapping> mappingBuilder, ElasticsearchClient client, ILogger logger = null) where T : class
+    public static ElasticMappingResolver Create<T>(Action<TypeMappingDescriptor<T>> mappingBuilder, ElasticsearchClient client, ILogger logger = null) where T : class
     {
         logger ??= NullLogger.Instance;
 
@@ -506,7 +542,7 @@ public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, Ty
         }, logger);
     }
 
-    public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, TypeMapping> mappingBuilder, ElasticsearchClient client, string index, ILogger logger = null) where T : class
+    public static ElasticMappingResolver Create<T>(Action<TypeMappingDescriptor<T>> mappingBuilder, ElasticsearchClient client, string index, ILogger logger = null) where T : class
     {
         logger ??= NullLogger.Instance;
 
@@ -521,10 +557,11 @@ public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, Ty
         }, logger);
     }
 
-    public static ElasticMappingResolver Create<T>(Func<TypeMappingDescriptor<T>, TypeMapping> mappingBuilder, Inferrer inferrer, Func<TypeMapping> getMapping, ILogger logger = null) where T : class
+    public static ElasticMappingResolver Create<T>(Action<TypeMappingDescriptor<T>> mappingBuilder, Inferrer inferrer, Func<TypeMapping> getMapping, ILogger logger = null) where T : class
     {
-        var codeMapping = mappingBuilder(new TypeMappingDescriptor<T>());
-        return new ElasticMappingResolver(codeMapping, inferrer, getMapping, logger: logger);
+        var descriptor = new TypeMappingDescriptor<T>();
+        mappingBuilder(descriptor);
+        return new ElasticMappingResolver(descriptor, inferrer, getMapping, logger: logger);
     }
 
 public static ElasticMappingResolver Create<T>(ElasticsearchClient client, ILogger logger = null)
@@ -562,6 +599,59 @@ public static ElasticMappingResolver Create(Func<TypeMapping> getMapping, Inferr
     {
         return new ElasticMappingResolver(getMapping, inferrer, logger: logger);
     }
+
+    #region Property Metadata
+
+    public IDictionary<string, object> GetPropertyMetadata(IProperty property)
+    {
+        if (property == null)
+            return null;
+
+        return _propertyMetadata.GetOrCreateValue(property);
+    }
+
+    public T GetPropertyMetadataValue<T>(IProperty property, string key, T defaultValue = default)
+    {
+        var metadata = GetPropertyMetadata(property);
+        if (metadata == null || !metadata.TryGetValue(key, out var value))
+            return defaultValue;
+
+        if (value is T typedValue)
+            return typedValue;
+
+        try
+        {
+            return (T)Convert.ChangeType(value, typeof(T));
+        }
+        catch
+        {
+            return defaultValue;
+        }
+    }
+
+    public void SetPropertyMetadataValue(IProperty property, string key, object value)
+    {
+        if (property == null)
+            return;
+
+        var metadata = _propertyMetadata.GetOrCreateValue(property);
+        metadata[key] = value;
+    }
+
+    public void CopyPropertyMetadata(IProperty source, IProperty target)
+    {
+        if (source == null || target == null)
+            return;
+
+        if (!_propertyMetadata.TryGetValue(source, out var sourceMetadata))
+            return;
+
+        var targetMetadata = _propertyMetadata.GetOrCreateValue(target);
+        foreach (var kvp in sourceMetadata)
+            targetMetadata[kvp.Key] = kvp.Value;
+    }
+
+    #endregion
 }
 
 public class FieldMapping

src/Foundatio.Parsers.ElasticQueries/ElasticQueryParser.cs

@@ -100,6 +100,12 @@ private void SetupQueryVisitorContextDefaults(IQueryVisitorContext context)
         if (Configuration.ValidationOptions != null && !context.HasValidationOptions())
             context.SetValidationOptions(Configuration.ValidationOptions);
 
+        if (context is IElasticQueryVisitorContext elasticVisitorContext)
+        {
+            if (Configuration.GeoLocationResolver != null && elasticVisitorContext.GeoLocationResolver == null)
+                elasticVisitorContext.GeoLocationResolver = Configuration.GeoLocationResolver;
+        }
+
         if (context.QueryType == QueryTypes.Query)
         {
             context.SetDefaultFields(Configuration.DefaultFields);
@@ -177,10 +183,22 @@ public async Task<Query> BuildQueryAsync(IQueryNode query, IElasticQueryVisitorC
         var q = await query.GetQueryAsync() ?? new MatchAllQuery();
         if (context?.UseScoring == false)
         {
-            q = new BoolQuery
+            // If the query is already a filter-only BoolQuery, don't wrap it again
+            // CombineQueriesVisitor now builds filter queries directly for filter mode
+            var boolQuery = q.Bool;
+            bool isFilterOnlyBoolQuery = boolQuery != null
+                && boolQuery.Filter?.Count > 0
+                && (boolQuery.Must == null || boolQuery.Must.Count == 0)
+                && (boolQuery.Should == null || boolQuery.Should.Count == 0)
+                && (boolQuery.MustNot == null || boolQuery.MustNot.Count == 0);
+
+            if (!isFilterOnlyBoolQuery)
             {
-                Filter = [q]
-            };
+                q = new BoolQuery
+                {
+                    Filter = [q]
+                };
+            }
         }
 
         return q;