diff --git a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs index a2a39c4b..f3b6b556 100644 --- a/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs +++ b/src/Speckle.Sdk/SQLite/SQLiteJsonCacheManager.cs @@ -9,84 +9,12 @@ namespace Speckle.Sdk.SQLite; public partial interface ISqLiteJsonCacheManager : IDisposable; [GenerateAutoInterface] -public sealed class SqLiteJsonCacheManager : ISqLiteJsonCacheManager +public sealed class SqLiteJsonCacheManager(ISqliteJsonCachePool pool, bool dispose) : ISqLiteJsonCacheManager { - private readonly CacheDbCommandPool _pool; - - public SqLiteJsonCacheManager(string path, int concurrency) - { - //disable pooling as we pool ourselves - var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path }; - _pool = new CacheDbCommandPool(builder.ToString(), concurrency); - Initialize(); - } - - [AutoInterfaceIgnore] - public void Dispose() => _pool.Dispose(); - - private void Initialize() - { - // NOTE: used for creating partioned object tables. - //string[] HexChars = new string[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" }; - //var cart = new List(); - //foreach (var str in HexChars) - // foreach (var str2 in HexChars) - // cart.Add(str + str2); - - _pool.Use(c => - { - const string COMMAND_TEXT = - @" - CREATE TABLE IF NOT EXISTS objects( - hash TEXT PRIMARY KEY, - content TEXT - ) WITHOUT ROWID; - "; - using (var command = new SqliteCommand(COMMAND_TEXT, c)) - { - command.ExecuteNonQuery(); - } - - // Insert Optimisations - - //Note / Hack: This setting has the potential to corrupt the db. - //cmd = new SqliteCommand("PRAGMA synchronous=OFF;", Connection); - //cmd.ExecuteNonQuery(); - - using (SqliteCommand cmd1 = new("PRAGMA count_changes=OFF;", c)) - { - cmd1.ExecuteNonQuery(); - } - - using (SqliteCommand cmd2 = new("PRAGMA temp_store=MEMORY;", c)) - { - cmd2.ExecuteNonQuery(); - } - - using (SqliteCommand cmd3 = new("PRAGMA mmap_size = 30000000000;", c)) - { - cmd3.ExecuteNonQuery(); - } - - using (SqliteCommand cmd4 = new("PRAGMA page_size = 32768;", c)) - { - cmd4.ExecuteNonQuery(); - } - - using (SqliteCommand cmd5 = new("PRAGMA journal_mode='wal';", c)) - { - cmd5.ExecuteNonQuery(); - } - //do this to wait 5 seconds to avoid db lock exceptions, this is 0 by default - using (SqliteCommand cmd6 = new("PRAGMA busy_timeout=5000;", c)) - { - cmd6.ExecuteNonQuery(); - } - }); - } + public ISqliteJsonCachePool Pool => pool; public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() => - _pool.Use( + pool.Use( CacheOperation.GetAll, command => { @@ -101,7 +29,7 @@ content TEXT ); public void DeleteObject(string id) => - _pool.Use( + pool.Use( CacheOperation.Delete, command => { @@ -111,7 +39,7 @@ public void DeleteObject(string id) => ); public string? GetObject(string id) => - _pool.Use( + pool.Use( CacheOperation.Get, command => { @@ -125,7 +53,7 @@ public void SaveObject(string id, string json) { id.NotNullOrWhiteSpace(); json.NotNullOrWhiteSpace(); - _pool.Use( + pool.Use( CacheOperation.InsertOrIgnore, command => { @@ -138,7 +66,7 @@ public void SaveObject(string id, string json) //This does an insert or replaces if already exists public void UpdateObject(string id, string json) => - _pool.Use( + pool.Use( CacheOperation.InsertOrReplace, command => { @@ -149,7 +77,7 @@ public void UpdateObject(string id, string json) => ); public void SaveObjects(IEnumerable<(string id, string json)> items) => - _pool.Use( + pool.Use( CacheOperation.BulkInsertOrIgnore, cmd => { @@ -195,7 +123,7 @@ private bool CreateBulkInsert(SqliteCommand cmd, IEnumerable<(string id, string } public bool HasObject(string objectId) => - _pool.Use( + pool.Use( CacheOperation.Has, command => { @@ -205,4 +133,13 @@ public bool HasObject(string objectId) => return rowFound; } ); + + [AutoInterfaceIgnore] + public void Dispose() + { + if (dispose) + { + pool.Dispose(); + } + } } diff --git a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs index f8fedca4..75869299 100644 --- a/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs +++ b/src/Speckle.Sdk/SQLite/SqLiteJsonCacheManagerFactory.cs @@ -1,19 +1,49 @@ -using Speckle.InterfaceGenerator; +using System.Collections.Concurrent; +using Speckle.InterfaceGenerator; using Speckle.Sdk.Logging; using Speckle.Sdk.Serialisation.Utilities; namespace Speckle.Sdk.SQLite; +public partial interface ISqLiteJsonCacheManagerFactory : IDisposable; + [GenerateAutoInterface] -public class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory +public sealed class SqLiteJsonCacheManagerFactory : ISqLiteJsonCacheManagerFactory { public const int INITIAL_CONCURRENCY = 4; - private ISqLiteJsonCacheManager Create(string path, int concurrency) => new SqLiteJsonCacheManager(path, concurrency); + private readonly ConcurrentDictionary _pools = new(); + + [AutoInterfaceIgnore] + public void Dispose() + { + foreach (var pool in _pools) + { + pool.Value.Dispose(); + } + + _pools.Clear(); + } + + private ISqliteJsonCachePool Create(string path, int concurrency) => new SqliteJsonCachePool(path, concurrency); public ISqLiteJsonCacheManager CreateForUser(string scope) => - Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1); + new SqLiteJsonCacheManager( +#pragma warning disable CA2000 + //this is fine because we told SqLiteJsonCacheManager to dispose this + Create(Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"), 1), +#pragma warning restore CA2000 + true + ); - public ISqLiteJsonCacheManager CreateFromStream(string streamId) => - Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); + public ISqLiteJsonCacheManager CreateFromStream(string streamId) + { + if (!_pools.TryGetValue(streamId, out var pool)) + { + pool = Create(SqlitePaths.GetDBPath(streamId), INITIAL_CONCURRENCY); + _pools.TryAdd(streamId, pool); + } + //never dispose pools for streams + return new SqLiteJsonCacheManager(pool, false); + } } diff --git a/src/Speckle.Sdk/SQLite/SqliteJsonCachePool.cs b/src/Speckle.Sdk/SQLite/SqliteJsonCachePool.cs new file mode 100644 index 00000000..a15ac9e2 --- /dev/null +++ b/src/Speckle.Sdk/SQLite/SqliteJsonCachePool.cs @@ -0,0 +1,99 @@ +using Microsoft.Data.Sqlite; +using Speckle.InterfaceGenerator; + +namespace Speckle.Sdk.SQLite; + +public partial interface ISqliteJsonCachePool : IDisposable +{ + T Use(CacheOperation type, Func handler); +} + +[GenerateAutoInterface] +public sealed class SqliteJsonCachePool : ISqliteJsonCachePool +{ + private readonly CacheDbCommandPool _pool; + + public SqliteJsonCachePool(string path, int concurrency) + { + Path = path; + Concurrency = concurrency; + //disable pooling as we pool ourselves + var builder = new SqliteConnectionStringBuilder { Pooling = false, DataSource = path }; + _pool = new CacheDbCommandPool(builder.ToString(), concurrency); + Initialize(); + } + + public string Path { get; } + public int Concurrency { get; } + + private void Initialize() + { + // NOTE: used for creating partioned object tables. + //string[] HexChars = new string[] { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" }; + //var cart = new List(); + //foreach (var str in HexChars) + // foreach (var str2 in HexChars) + // cart.Add(str + str2); + + _pool.Use(c => + { + const string COMMAND_TEXT = + @" + CREATE TABLE IF NOT EXISTS objects( + hash TEXT PRIMARY KEY, + content TEXT + ) WITHOUT ROWID; + "; + using (var command = new SqliteCommand(COMMAND_TEXT, c)) + { + command.ExecuteNonQuery(); + } + + // Insert Optimisations + + //Note / Hack: This setting has the potential to corrupt the db. + //cmd = new SqliteCommand("PRAGMA synchronous=OFF;", Connection); + //cmd.ExecuteNonQuery(); + + using (SqliteCommand cmd1 = new("PRAGMA count_changes=OFF;", c)) + { + cmd1.ExecuteNonQuery(); + } + + using (SqliteCommand cmd2 = new("PRAGMA temp_store=MEMORY;", c)) + { + cmd2.ExecuteNonQuery(); + } + + using (SqliteCommand cmd3 = new("PRAGMA mmap_size = 30000000000;", c)) + { + cmd3.ExecuteNonQuery(); + } + + using (SqliteCommand cmd4 = new("PRAGMA page_size = 32768;", c)) + { + cmd4.ExecuteNonQuery(); + } + + using (SqliteCommand cmd5 = new("PRAGMA journal_mode='wal';", c)) + { + cmd5.ExecuteNonQuery(); + } + //do this to wait 5 seconds to avoid db lock exceptions, this is 0 by default + using (SqliteCommand cmd6 = new("PRAGMA busy_timeout=5000;", c)) + { + cmd6.ExecuteNonQuery(); + } + }); + } + + public void Use(Action handler) => _pool.Use(handler); + + public void Use(CacheOperation type, Action handler) => _pool.Use(type, handler); + + [AutoInterfaceIgnore] + public T Use(CacheOperation type, Func handler) => _pool.Use(type, handler); + + [AutoInterfaceIgnore] + public void Dispose() => _pool.Dispose(); +} diff --git a/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs b/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs index 26e142be..c5e48cb0 100644 --- a/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs +++ b/src/Speckle.Sdk/Serialisation/V2/MemoryJsonCacheManager.cs @@ -7,6 +7,9 @@ namespace Speckle.Sdk.Serialisation.V2; public class MemoryJsonCacheManager(ConcurrentDictionary jsonCache) : ISqLiteJsonCacheManager #pragma warning restore CA1063 { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 public IReadOnlyCollection<(string Id, string Json)> GetAllObjects() => jsonCache.Select(x => (x.Key.Value, x.Value.Value)).ToList(); diff --git a/src/Speckle.Sdk/ServiceRegistration.cs b/src/Speckle.Sdk/ServiceRegistration.cs index 8f8066ae..0659a29e 100644 --- a/src/Speckle.Sdk/ServiceRegistration.cs +++ b/src/Speckle.Sdk/ServiceRegistration.cs @@ -94,9 +94,12 @@ SpeckleSdkOptions speckleSdkOptions typeof(DeserializeProcess), typeof(ObjectLoader), typeof(TraversalRule), - typeof(Client) + typeof(Client), + typeof(SqliteJsonCachePool) ); serviceCollection.AddMatchingInterfacesAsTransient(typeof(GraphQLRetry).Assembly); + //we want to make sqlite pools be singletons per stream so needs a singleton factory + serviceCollection.AddSingleton(); return serviceCollection; } diff --git a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs index 8b278d59..dba2cd71 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DetachedTests.cs @@ -373,6 +373,9 @@ CancellationToken cancellationToken public class DummySendCacheManager(Dictionary objects) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 public void Dispose() { } public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs b/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs index 2d6ca588..1558c9c6 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/DummyCancellationSqLiteSendManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Serialization.Tests; public class DummyCancellationSqLiteSendManager : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 public string? GetObject(string id) => null; public void SaveObject(string id, string json) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs b/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs index 5572aa02..d8707d82 100644 --- a/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs +++ b/tests/Speckle.Sdk.Serialization.Tests/Framework/ExceptionSendCacheManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Serialization.Tests.Framework; public class ExceptionSendCacheManager(bool? hasObject = null, int? exceptionsAfter = null) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 private readonly object _lock = new(); private int _count; diff --git a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs index fcfe6832..8307d88f 100644 --- a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs +++ b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteReceiveManager.cs @@ -5,6 +5,9 @@ namespace Speckle.Sdk.Testing.Framework; public sealed class DummySqLiteReceiveManager(IReadOnlyDictionary savedObjects) : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 public void Dispose() { } public IReadOnlyCollection<(string, string)> GetAllObjects() => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs index b1425dd6..b3564cee 100644 --- a/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs +++ b/tests/Speckle.Sdk.Testing/Framework/DummySqLiteSendManager.cs @@ -4,6 +4,9 @@ namespace Speckle.Sdk.Testing.Framework; public class DummySqLiteSendManager : ISqLiteJsonCacheManager { +#pragma warning disable CA1065 + public ISqliteJsonCachePool Pool => throw new NotImplementedException(); +#pragma warning restore CA1065 public string? GetObject(string id) => throw new NotImplementedException(); public void SaveObject(string id, string json) => throw new NotImplementedException(); diff --git a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs index 7dfb71f2..f2b818dd 100644 --- a/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs +++ b/tests/Speckle.Sdk.Tests.Unit/SQLite/SQLiteJsonCacheManagerTests.cs @@ -22,7 +22,7 @@ public void Dispose() public void TestGetAll() { var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") }; - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.SaveObjects(data); var items = manager.GetAllObjects(); items.Count.Should().Be(data.Count); @@ -38,7 +38,7 @@ public void TestGetAll() public void TestGet() { var data = new List<(string id, string json)>() { ("id1", "1"), ("id2", "2") }; - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); foreach (var d in data) { manager.SaveObject(d.id, d.json); @@ -84,7 +84,7 @@ public void TestGet() public void TestLargeJsonPayload() { var largeJson = new string('a', 100_000); - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.SaveObject("large", largeJson); var result = manager.GetObject("large"); result.Should().Be(largeJson); @@ -96,7 +96,7 @@ public void TestSpecialCharactersInIdAndJson() var id = "spécial_字符_!@#$%^&*()"; var json = /*lang=json,strict*/ "{\"value\": \"特殊字符!@#$%^&*()\"}"; - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.SaveObject(id, json); var result = manager.GetObject(id); result.Should().Be(json); @@ -108,7 +108,7 @@ public void TestSpecialCharactersInIdAndJson() [Fact] public void TestBulkInsertEmptyCollection() { - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.SaveObjects(new List<(string, string)>()); manager.GetAllObjects().Count.Should().Be(0); } @@ -116,7 +116,7 @@ public void TestBulkInsertEmptyCollection() [Fact] public void TestRepeatedUpdateAndDelete() { - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.SaveObject("id", "1"); manager.UpdateObject("id", "2"); manager.UpdateObject("id", "3"); @@ -129,7 +129,7 @@ public void TestRepeatedUpdateAndDelete() [Fact] public void TestGetAndDeleteNonExistentId() { - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); manager.GetObject("doesnotexist").Should().BeNull(); manager.HasObject("doesnotexist").Should().BeFalse(); manager.DeleteObject("doesnotexist"); // Should not throw @@ -138,7 +138,7 @@ public void TestGetAndDeleteNonExistentId() [Fact] public void TestNullOrEmptyInput() { - using var manager = new SqLiteJsonCacheManager(_basePath, 2); + using var manager = new SqLiteJsonCacheManager(new SqliteJsonCachePool(_basePath, 2), true); // Empty id Assert.Throws(() => manager.SaveObject("", "emptyid")); // Empty json diff --git a/tests/Speckle.Sdk.Tests.Unit/SQLite/SqLiteJsonCacheManagerFactoryTests.cs b/tests/Speckle.Sdk.Tests.Unit/SQLite/SqLiteJsonCacheManagerFactoryTests.cs new file mode 100644 index 00000000..67d7b46b --- /dev/null +++ b/tests/Speckle.Sdk.Tests.Unit/SQLite/SqLiteJsonCacheManagerFactoryTests.cs @@ -0,0 +1,86 @@ +using FluentAssertions; +using Speckle.Sdk.Logging; +using Speckle.Sdk.Serialisation.Utilities; +using Speckle.Sdk.SQLite; + +namespace Speckle.Sdk.Tests.Unit.SQLite; + +public class SqLiteJsonCacheManagerFactoryTests +{ + [Fact] + public void CreateForUser_ShouldReturnManagerWithCorrectPathAndConcurrency() + { + // Arrange + var scope = "testuser"; + var expectedPath = Path.Combine(SpecklePathProvider.UserApplicationDataPath(), "Speckle", $"{scope}.db"); + + // Act + using (var factory = new SqLiteJsonCacheManagerFactory()) + { + using (var manager = factory.CreateForUser(scope)) + { + // Assert + manager.Should().NotBeNull(); + manager.Pool.Path.Should().Be(expectedPath); + manager.Pool.Concurrency.Should().Be(1); + } + } + + // Cleanup + if (File.Exists(expectedPath)) + { + File.Delete(expectedPath); + } + } + + [Fact] + public void CreateFromStream_ShouldReturnManagerWithCorrectPathAndConcurrency_AndCleanup() + { + // Arrange + var streamId = "stream123"; + var expectedPath = SqlitePaths.GetDBPath(streamId); + using (var factory = new SqLiteJsonCacheManagerFactory()) + { + // Act + using (var manager = factory.CreateFromStream(streamId)) + { + // Assert + manager.Should().NotBeNull(); + manager.Pool.Path.Should().Be(expectedPath); + manager.Pool.Concurrency.Should().Be(SqLiteJsonCacheManagerFactory.INITIAL_CONCURRENCY); + } + } + + // Cleanup + if (File.Exists(expectedPath)) + { + File.Delete(expectedPath); + } + } + + [Fact] + public void CreateFromStream_ShouldReturnCachedManagerForSameStreamId_AndCleanup() + { + var streamId = "stream123"; + var expectedPath = SqlitePaths.GetDBPath(streamId); + // Arrange + using (var factory = new SqLiteJsonCacheManagerFactory()) + { + // Act + using (var manager1 = factory.CreateFromStream(streamId)) + { + using var manager2 = factory.CreateFromStream(streamId); + + // Assert + manager1.Should().NotBeSameAs(manager2); + manager1.Pool.Should().BeSameAs(manager2.Pool); + } + } + + // Cleanup + if (File.Exists(expectedPath)) + { + File.Delete(expectedPath); + } + } +}