Orleans Indexing and Lucene.Net

Directory.Build.props

@@ -58,17 +58,17 @@
     <MicrosoftBuildVersion>15.7.179</MicrosoftBuildVersion>
     <MicrosoftCodeAnalysisVersion>2.8.2</MicrosoftCodeAnalysisVersion>
 
-    <MicrosoftExtensionsConfigurationBinderVersion>2.1.1</MicrosoftExtensionsConfigurationBinderVersion>
-    <MicrosoftExtensionsConfigurationEnvironmentVariablesVersion>2.1.1</MicrosoftExtensionsConfigurationEnvironmentVariablesVersion>
-    <MicrosoftExtensionsConfigurationJsonVersion>2.1.1</MicrosoftExtensionsConfigurationJsonVersion>
-    <MicrosoftExtensionsConfigurationVersion>2.1.1</MicrosoftExtensionsConfigurationVersion>
-    <MicrosoftExtensionsDependencyInjectionVersion>2.1.1</MicrosoftExtensionsDependencyInjectionVersion>
-    <MicrosoftExtensionsDependencyModelVersion>2.0.0</MicrosoftExtensionsDependencyModelVersion>
-    <MicrosoftExtensionsLoggingVersion>2.0.0</MicrosoftExtensionsLoggingVersion>
-    <MicrosoftExtensionsOptionsConfigurationExtensionsVersion>2.1.1</MicrosoftExtensionsOptionsConfigurationExtensionsVersion>
-    <MicrosoftExtensionsOptionsVersion>2.1.1</MicrosoftExtensionsOptionsVersion>
-    <MicrosoftExtensionsHostingAbstractionsVersion>2.1.0</MicrosoftExtensionsHostingAbstractionsVersion>
-    <MicrosoftExtensionsHostingVersion>2.1.0</MicrosoftExtensionsHostingVersion>
+    <MicrosoftExtensionsConfigurationBinderVersion>3.0.0</MicrosoftExtensionsConfigurationBinderVersion>
+    <MicrosoftExtensionsConfigurationEnvironmentVariablesVersion>3.0.0</MicrosoftExtensionsConfigurationEnvironmentVariablesVersion>
+    <MicrosoftExtensionsConfigurationJsonVersion>3.0.0</MicrosoftExtensionsConfigurationJsonVersion>
+    <MicrosoftExtensionsConfigurationVersion>3.0.0</MicrosoftExtensionsConfigurationVersion>
+    <MicrosoftExtensionsDependencyInjectionVersion>3.0.0</MicrosoftExtensionsDependencyInjectionVersion>
+    <MicrosoftExtensionsDependencyModelVersion>3.0.0</MicrosoftExtensionsDependencyModelVersion>
+    <MicrosoftExtensionsLoggingVersion>3.0.0</MicrosoftExtensionsLoggingVersion>
+    <MicrosoftExtensionsOptionsConfigurationExtensionsVersion>3.0.0</MicrosoftExtensionsOptionsConfigurationExtensionsVersion>
+    <MicrosoftExtensionsOptionsVersion>3.0.0</MicrosoftExtensionsOptionsVersion>
+    <MicrosoftExtensionsHostingAbstractionsVersion>3.0.0</MicrosoftExtensionsHostingAbstractionsVersion>
+    <MicrosoftExtensionsHostingVersion>3.0.0</MicrosoftExtensionsHostingVersion>
 
     <MicrosoftApplicationInsightsVersion>2.4.0</MicrosoftApplicationInsightsVersion>
     <MicrosoftAzureEventHubsVersion>2.2.1</MicrosoftAzureEventHubsVersion>
@@ -104,7 +104,7 @@
     <DotNetxUnitVersion>2.3.1</DotNetxUnitVersion>
     <xUnitVersion>2.4.1</xUnitVersion>
     <NodaTimeVersion>2.2.2</NodaTimeVersion>
-    <OrleansVersion>3.0.0-beta*</OrleansVersion>
+    <OrleansVersion>3.4.2</OrleansVersion>
 
     <!-- Tooling related packages -->
     <SourceLinkVersion>2.7.6</SourceLinkVersion>

src/Orleans.Indexing/Core/Utils/ApplicationPartsIndexableGrainLoader.cs

@@ -50,7 +50,7 @@ internal IndexRegistry CreateIndexRegistry()
         /// registered to the IServiceCollection before Silo construction and the rest of Index creation requires the IServiceProvider which
         /// is created as part of Silo construction.</remarks>
         /// <returns>An index registry for the silo. </returns>
-        internal static void RegisterGrainServices(HostBuilderContext context, IServiceCollection services, IndexingOptions indexingOptions)
+        internal static void RegisterGrainServices(Microsoft.Extensions.Hosting.HostBuilderContext context, IServiceCollection services, IndexingOptions indexingOptions)
         {
             var indexedClasses = new HashSet<Type>();
             var indexedInterfaces = new HashSet<Type>();
@@ -189,7 +189,7 @@ bool isInDict(string propName)
 
             foreach (var indexableBaseInterface in indexableBaseInterfaces)
             {
-                // ... and its generic argument is a class (TProperties)... 
+                // ... and its generic argument is a class (TProperties)...
                 var propertiesClassType = indexableBaseInterface.GetGenericArguments()[0];
                 if (propertiesClassType.GetTypeInfo().IsClass)
                 {

src/Orleans.Indexing/Extensions/PersistentStateIndexExtensions.cs

@@ -0,0 +1,32 @@
+using System.Threading.Tasks;
+using Orleans.Runtime;
+
+namespace Orleans.Indexing
+{
+
+    public static class PersistentStateIndexExtensions
+    {
+
+        public static Task WriteIndexAsync<TState>(this IPersistentState<TState> state)
+        {
+            return Task.CompletedTask;
+        }
+
+        public static Task ClearIndexAsync<TState>(this IPersistentState<TState> state)
+        {
+            return Task.CompletedTask;
+        }
+
+        public static Task WriteActiveIndexAsync<TState>(this IPersistentState<TState> state)
+        {
+            return Task.CompletedTask;
+        }
+
+        public static Task ClearActiveIndexAsync<TState>(this IPersistentState<TState> state)
+        {
+            return Task.CompletedTask;
+        }
+
+    }
+
+}

src/Orleans.Indexing/Hosting/SiloBuilderExtensions.cs

@@ -14,9 +14,11 @@ public static class SiloBuilderExtensions
         /// <summary>
         /// Configure silo to use indexing using a configure action.
         /// </summary>
-        public static ISiloHostBuilder UseIndexing(this ISiloHostBuilder builder, Action<IndexingOptions> configureOptions = null)
+        public static ISiloBuilder UseIndexing(this ISiloBuilder builder, Action<IndexingOptions> configureOptions = null)
         {
+
             // This is necessary to get the configured NumWorkflowQueuesPerInterface for IndexFactory.RegisterIndexWorkflowQueueGrainServices.
+
             var indexingOptions = new IndexingOptions();
             configureOptions?.Invoke(indexingOptions);
 
@@ -28,7 +30,7 @@ public static ISiloHostBuilder UseIndexing(this ISiloHostBuilder builder, Action
                 .AddMemoryGrainStorage(IndexingConstants.MEMORY_STORAGE_PROVIDER_NAME)
                 .ConfigureApplicationParts(parts => parts.AddFrameworkPart(typeof(SiloBuilderExtensions).Assembly))
                 .ConfigureServices(services => services.UseIndexing(indexingOptions))
-                .ConfigureServices((context, services) => ApplicationPartsIndexableGrainLoader.RegisterGrainServices(context, services, indexingOptions))
+                //.ConfigureServices((context, services) => ApplicationPartsIndexableGrainLoader.RegisterGrainServices(context, services, indexingOptions))
                 .UseTransactions();
         }
 

src/Orleans.Indexing/Orleans.Indexing.csproj

@@ -24,6 +24,11 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Lucene.Net" Version="4.8.0-beta00014" />
+    <PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00014" />
+    <PackageReference Include="Lucene.Net.Queries" Version="4.8.0-beta00014" />
+    <PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00014" />
+    <PackageReference Include="Lucene.Net.Spatial" Version="4.8.0-beta00014" />
     <PackageReference Include="Microsoft.Orleans.Core" Version="$(OrleansVersion)" />
     <PackageReference Include="Microsoft.Orleans.OrleansCodeGenerator.Build" Version="$(OrleansVersion)">
       <PrivateAssets>all</PrivateAssets>

src/Orleans.Indexing/newversion/IndexService.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq.Expressions;
+using System.Threading.Tasks;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.QueryParsers.Classic;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+using Orleans.Concurrency;
+using Orleans.Placement;
+using Orleans.Runtime;
+using Orleans.Services;
+using Directory = System.IO.Directory;
+
+namespace Orleans.Indexing
+{
+
+    public class GrainDocument
+    {
+        public static string GrainIdFieldName = "___grainId";
+        public GrainDocument(string grainId)
+        {
+            this.LuceneDocument = new Document();
+            this.LuceneDocument.Add(new StringField(GrainIdFieldName, grainId, Field.Store.NO));
+        }
+        public Document LuceneDocument { get; }
+    }
+
+    public interface IIndexService : IGrainService
+    {
+
+    }
+
+
+    public interface IIndexGrain : IGrainWithStringKey
+    {
+        Task WriteIndex(GrainDocument document);
+        Task<TopDocs> QueryByField(string field, string query, int take = 1000);
+    }
+
+
+    [Reentrant]
+    public class IndexService : GrainService, IIndexService
+    {
+
+    }
+
+    [PreferLocalPlacement]
+    public class IndexGrain : Grain, IIndexGrain
+    {
+        // Ensures index backward compatibility
+        private const LuceneVersion AppLuceneVersion = LuceneVersion.LUCENE_48;
+        // private static string indexPath = "indexPath";
+        private BaseDirectory indexDirectory;
+        private DirectoryReader directoryReader;
+        private Analyzer analyzer;
+        private IndexWriter indexWriter;
+        private IndexSearcher indexSearcher;
+
+        public override Task OnActivateAsync()
+        {
+            this.indexDirectory = GetDirectory();
+            this.analyzer = new StandardAnalyzer(AppLuceneVersion);
+            var config = new IndexWriterConfig(AppLuceneVersion, this.analyzer);
+            this.indexWriter = new IndexWriter(this.indexDirectory, config);
+            this.indexWriter.Commit();
+
+            this.directoryReader = DirectoryReader.Open(this.indexDirectory);
+            this.indexSearcher = new IndexSearcher(this.directoryReader);
+
+            return Task.CompletedTask;
+        }
+
+        public override Task OnDeactivateAsync()
+        {
+            this.indexWriter?.Dispose();
+            this.analyzer?.Dispose();
+            this.directoryReader?.Dispose();
+            return Task.CompletedTask;
+        }
+
+        private BaseDirectory GetDirectory()
+        {
+            return new RAMDirectory();
+            // return FSDirectory.Open(indexPath);
+        }
+
+        public Task WriteIndex(GrainDocument document) => Task.Run(() =>
+        {
+            var parser = new QueryParser(AppLuceneVersion, GrainDocument.GrainIdFieldName, this.analyzer);
+            var query = parser.Parse(document.LuceneDocument.GetField(GrainDocument.GrainIdFieldName).GetStringValue());
+            this.indexWriter.DeleteDocuments(query);
+            this.indexWriter.AddDocument(document.LuceneDocument);
+            this.indexWriter.Commit();
+
+            this.directoryReader = DirectoryReader.OpenIfChanged(this.directoryReader) ?? this.directoryReader;
+            this.indexSearcher = new IndexSearcher(this.directoryReader);
+
+            return Task.CompletedTask;
+        });
+
+        public Task<TopDocs> QueryByField(string field, string query, int take = 1000) => Task.Run(() =>
+        {
+            var parser = new QueryParser(AppLuceneVersion, GrainDocument.GrainIdFieldName, this.analyzer);
+            var result = this.indexSearcher.Search(parser.Parse(query), null, take);
+            return result;
+        });
+    }
+}

test/Orleans.Indexing.Tests/BaseIndexingFixture.cs

@@ -7,6 +7,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
+using Microsoft.Extensions.Hosting;
 using TestExtensions;
 
 namespace Orleans.Indexing.Tests
@@ -20,18 +21,8 @@ protected TestClusterBuilder ConfigureTestClusterForIndexing(TestClusterBuilder
             return builder;
         }
 
-        internal static ISiloHostBuilder Configure(ISiloHostBuilder hostBuilder, string databaseName = null)
+        internal static ISiloBuilder Configure(ISiloBuilder hostBuilder, string databaseName = null)
         {
-            string cosmosDBEndpoint = string.Empty, cosmosDBKey = string.Empty;
-            if (databaseName != null)
-            {
-                if (!TestDefaultConfiguration.GetValue("CosmosDBEndpoint", out cosmosDBEndpoint)
-                    || !TestDefaultConfiguration.GetValue("CosmosDBKey", out cosmosDBKey))
-                {
-                    throw new IndexConfigurationException("CosmosDB connection values are not specified");
-                }
-            }
-
             hostBuilder.AddMemoryGrainStorage(IndexingTestConstants.GrainStore)
                        .AddMemoryGrainStorage("PubSubStore") // PubSubStore service is needed for the streams underlying OrleansQueryResults
                        .ConfigureLogging(loggingBuilder =>
@@ -43,20 +34,7 @@ internal static ISiloHostBuilder Configure(ISiloHostBuilder hostBuilder, string
                        {
                            parts.AddApplicationPart(typeof(BaseIndexingFixture).Assembly).WithReferences();
                        });
-            return databaseName != null
-                ? hostBuilder.AddCosmosDBGrainStorage(IndexingTestConstants.CosmosDBGrainStorage, opt =>
-                    {
-                        opt.AccountEndpoint = cosmosDBEndpoint;
-                        opt.AccountKey = cosmosDBKey;
-                        opt.ConnectionMode = Microsoft.Azure.Documents.Client.ConnectionMode.Gateway;
-                        opt.DropDatabaseOnInit = true;
-                        opt.AutoUpdateStoredProcedures = true;
-                        opt.CanCreateResources = true;
-                        opt.DB = databaseName;
-                        opt.InitStage = ServiceLifecycleStage.RuntimeStorageServices;
-                        opt.StateFieldsToIndex.AddRange(GetDSMIStateFieldsToIndex());
-                    })
-                : hostBuilder;
+            return hostBuilder;
         }
 
         internal static IClientBuilder Configure(IClientBuilder clientBuilder)
@@ -71,35 +49,5 @@ internal static IClientBuilder Configure(IClientBuilder clientBuilder)
                                     parts.AddApplicationPart(typeof(BaseIndexingFixture).Assembly);
                                 });
         }
-
-        // Code below adapted from ApplicationPartsIndexableGrainLoader to identify the necessary fields for the DSMI storage
-        // provider to index.
-
-        private static IEnumerable<string> GetDSMIStateFieldsToIndex()
-        {
-            var grainClassTypes = typeof(BaseIndexingFixture).Assembly.GetConcreteGrainClasses().ToArray();
-
-            // Orleans.CosmosDB appends the field names to "State."; thus we do not prepend the interface names.
-            var interfacesToIndexedPropertyNames = new Dictionary<Type, string[]>();
-            foreach (var grainClassType in grainClassTypes)
-            {
-                GetDSMIFieldsForASingleGrainType(grainClassType, interfacesToIndexedPropertyNames);
-            }
-            return new HashSet<string>(interfacesToIndexedPropertyNames.Where(kvp => kvp.Value.Length > 0).SelectMany(kvp => kvp.Value));
-        }
-
-        internal static void GetDSMIFieldsForASingleGrainType(Type grainClassType, Dictionary<Type, string[]> interfacesToIndexedPropertyNames)
-        {
-            foreach (var (grainInterfaceType, propertiesClassType) in ApplicationPartsIndexableGrainLoader.EnumerateIndexedInterfacesForAGrainClassType(grainClassType)
-                                                                        .Where(tup => !interfacesToIndexedPropertyNames.ContainsKey(tup.interfaceType)))
-            {
-                // TODO: See comments in DSMIGrain.LookupGrainReferences; get the path with and without the transactional storage wrapper prefix.
-                interfacesToIndexedPropertyNames[grainInterfaceType] = propertiesClassType.GetProperties()
-                                                                        .Where(propInfo => propInfo.GetCustomAttributes<StorageManagedIndexAttribute>(inherit: false).Any())
-                                                                        .Select(propInfo => IndexingConstants.UserStatePrefix + propInfo.Name)
-                                                                        .SelectMany(path => new[] {path, $"{nameof(TransactionalStateRecord<object>.CommittedState)}.{path}"})
-                                                                        .ToArray();
-            }
-        }
     }
 }

test/Orleans.Indexing.Tests/BaseIndexingTestRunnerBase.cs

@@ -0,0 +1,59 @@
+using Xunit.Abstractions;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.DependencyInjection;
+using Orleans.TestingHost;
+using System.Threading.Tasks;
+using System.Linq;
+using System;
+using Xunit;
+
+namespace Orleans.Indexing.Tests
+{
+
+    public class BaseIndexingTestRunnerBase : IDisposable
+    {
+        private BaseIndexingFixture fixture;
+
+        internal readonly ITestOutputHelper Output;
+        internal IClusterClient ClusterClient => this.fixture.Client;
+
+        internal IGrainFactory GrainFactory => this.fixture.GrainFactory;
+
+
+        internal ILoggerFactory LoggerFactory { get; }
+
+        protected TestCluster HostedCluster => this.fixture.HostedCluster;
+
+        protected BaseIndexingTestRunnerBase(BaseIndexingFixture fixture, ITestOutputHelper output)
+        {
+            this.fixture = fixture;
+            this.Output = output;
+            this.LoggerFactory = this.ClusterClient.ServiceProvider.GetRequiredService<ILoggerFactory>();
+        }
+
+        protected TInterface GetGrain<TInterface>(long primaryKey) where TInterface : IGrainWithIntegerKey
+            => this.GrainFactory.GetGrain<TInterface>(primaryKey);
+
+        protected TInterface GetGrain<TInterface, TImplClass>(long primaryKey) where TInterface : IGrainWithIntegerKey
+            => this.GetGrain<TInterface>(primaryKey, typeof(TImplClass));
+
+        protected TInterface GetGrain<TInterface>(long primaryKey, Type grainImplType) where TInterface : IGrainWithIntegerKey
+            => this.GrainFactory.GetGrain<TInterface>(primaryKey, grainImplType.FullName.Replace("+", "."));
+
+
+        protected Task StartAndWaitForSecondSilo()
+        {
+            if (this.HostedCluster.SecondarySilos.Count == 0)
+            {
+                this.HostedCluster.StartAdditionalSilo();
+                return this.HostedCluster.WaitForLivenessToStabilizeAsync();
+            }
+            return Task.CompletedTask;
+        }
+
+        public void Dispose()
+        {
+            this.HostedCluster.StopAllSilos();
+        }
+    }
+}