Exposing S3 connector

.github/workflows/build_and_test.yml

@@ -40,11 +40,11 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           sudo apt update
-          sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev build-essential cmake pkg-config
+          sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev build-essential cmake pkg-config libcurl4-openssl-dev libssl-dev
 
       - name: Install dependencies (macOS)
         if: runner.os == 'macOS'
-        run: brew install zstd lz4 snappy
+        run: brew install zstd lz4 snappy curl openssl
 
       - name: Setup MSYS2 (Windows)
         if: runner.os == 'Windows'
@@ -65,7 +65,7 @@ jobs:
         if: runner.os == 'Linux'
         run: |
           cd tidesdb
-          cmake -S . -B build -DTIDESDB_BUILD_TESTS=OFF -DTIDESDB_WITH_SANITIZER=OFF -DBUILD_SHARED_LIBS=ON
+          cmake -S . -B build -DTIDESDB_BUILD_TESTS=OFF -DTIDESDB_WITH_SANITIZER=OFF -DBUILD_SHARED_LIBS=ON -DTIDESDB_WITH_S3=ON
           cmake --build build --config Release
           sudo cmake --install build
           sudo ldconfig
@@ -79,6 +79,7 @@ jobs:
             -DTIDESDB_BUILD_TESTS=OFF \
             -DTIDESDB_WITH_SANITIZER=OFF \
             -DBUILD_SHARED_LIBS=ON \
+            -DTIDESDB_WITH_S3=ON \
             -DCMAKE_PREFIX_PATH="${HOMEBREW_PREFIX}"
           cmake --build build --config Release
           sudo cmake --install build
@@ -149,6 +150,7 @@ jobs:
             -DTIDESDB_WITH_SANITIZER=OFF \
             -DTIDESDB_BUILD_TESTS=OFF \
             -DBUILD_SHARED_LIBS=ON \
+            -DTIDESDB_WITH_S3=ON \
             -DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=ON \
             -S . -B build
           cmake --build build --config Release

src/TidesDB/Native/NativeMethods.cs

@@ -216,6 +216,10 @@ internal static partial class NativeMethods
     [LibraryImport(LibraryName, EntryPoint = "tidesdb_objstore_fs_create", StringMarshalling = StringMarshalling.Utf8)]
     internal static partial nint tidesdb_objstore_fs_create(string rootDir);
 
+    // Object store operations
+    [LibraryImport(LibraryName, EntryPoint = "tidesdb_objstore_s3_create", StringMarshalling = StringMarshalling.Utf8)]
+    internal static partial nint tidesdb_objstore_s3_create(string endpoint, string bucket, string? prefix, string accessKey, string secretKey, string? region, int useSsl, int usePathStyle);
+
     [LibraryImport(LibraryName, EntryPoint = "tidesdb_objstore_default_config")]
     internal static partial NativeObjStoreConfig tidesdb_objstore_default_config();
 

src/TidesDB/TidesDB.cs

@@ -63,7 +63,15 @@ public static TidesDb Open(Config config)
                         NativeMethods.tidesdb_objstore_fs_create(
                             osCfg.FsRootDir ?? throw new ArgumentException("FsRootDir is required for filesystem connector")),
                     ObjectStoreConnectorType.S3 =>
-                        throw new NotSupportedException("S3 connector requires native tidesdb_objstore_s3_create which is not yet exposed in the C# binding. Use the C API directly or contribute S3 support."),
+                        NativeMethods.tidesdb_objstore_s3_create(
+                            osCfg.S3Endpoint ?? throw new ArgumentException("S3Endpoint is required for S3 connector"), 
+                            osCfg.S3Bucket ?? throw new ArgumentException("S3Bucket is required for S3 connector"), 
+                            osCfg.S3KeyPrefix, 
+                            osCfg.S3AccessKey ?? throw new ArgumentException("S3AccessKey is required for S3 connector"), 
+                            osCfg.S3SecretKey ?? throw new ArgumentException("S3SecretKey is required for S3 connector"), 
+                            osCfg.S3Region, 
+                            osCfg.S3UseSsl ? 1 : 0, 
+                            osCfg.S3UsePathStyle ? 1 : 0),
                     _ => throw new ArgumentException($"Unknown connector type: {osCfg.ConnectorType}")
                 };
 

tests/TidesDB.Tests/TidesDBTests.cs

@@ -1321,6 +1321,59 @@ public void OpenWithObjectStore_Filesystem_ShouldSucceed()
         }
     }
 
+    [Fact]
+    public void OpenWithObjectStore_S3_ShouldSucceed()
+    {
+        var objStoreDir = Path.Combine(Path.GetTempPath(), $"tidesdb_objstore_{Guid.NewGuid()}");
+        Directory.CreateDirectory(objStoreDir);
+
+        try
+        {
+            var config = new Config
+            {
+                DbPath = _testDbPath,
+                NumFlushThreads = 1,
+                NumCompactionThreads = 1,
+                LogLevel = LogLevel.Info,
+                BlockCacheSize = 64 * 1024 * 1024,
+                MaxOpenSstables = 256,
+                ObjectStoreConfig = new ObjectStoreConfig
+                {
+                    ConnectorType = ObjectStoreConnectorType.S3,
+                    S3Endpoint = "http://localhost:9000",
+                    S3Bucket = "my-tidesdb",
+                    S3AccessKey = "IoCMWHmye5ZjHoSwTIok",
+                    S3SecretKey = "YECnEaIco4s68zH3jqZ6HKlyC8FCZq5k1Ue4MLiq",
+                    LocalCacheMaxBytes = 128 * 1024 * 1024,
+                    MaxConcurrentUploads = 4,
+                    MaxConcurrentDownloads = 8,
+                },
+            };
+
+            using var db = TidesDb.Open(config);
+            _db = db;
+            Assert.NotNull(db);
+
+            db.CreateColumnFamily("test_cf");
+            var cf = db.GetColumnFamily("test_cf");
+            Assert.NotNull(cf);
+
+            using var txn = db.BeginTransaction();
+            txn.Put(cf, Encoding.UTF8.GetBytes("key1"), Encoding.UTF8.GetBytes("value1"));
+            txn.Commit();
+
+            using var readTxn = db.BeginTransaction();
+            var result = readTxn.Get(cf, Encoding.UTF8.GetBytes("key1"));
+            Assert.NotNull(result);
+            Assert.Equal("value1", Encoding.UTF8.GetString(result));
+        }
+        finally
+        {
+            if (Directory.Exists(objStoreDir))
+                Directory.Delete(objStoreDir, true);
+        }
+    }
+
     [Fact]
     public void OpenWithObjectStore_DbStats_ShouldShowObjectStoreEnabled()
     {
@@ -1374,6 +1427,76 @@ public void ObjectStoreConfig_RequiresFsRootDir()
         Assert.Throws<ArgumentException>(() => TidesDb.Open(config));
     }
 
+    [Fact]
+    public void ObjectStoreConfig_RequiresS3Endpoint()
+    {
+        var config = new Config
+        {
+            DbPath = _testDbPath,
+            ObjectStoreConfig = new ObjectStoreConfig
+            {
+                ConnectorType = ObjectStoreConnectorType.S3,
+                // S3Endpoint = "http://localhost:9000",  
+            },
+        };
+
+        Assert.Throws<ArgumentException>(() => TidesDb.Open(config));
+    }
+
+    [Fact]
+    public void ObjectStoreConfig_RequiresS3Bucket()
+    {
+        var config = new Config
+        {
+            DbPath = _testDbPath,
+            ObjectStoreConfig = new ObjectStoreConfig
+            {
+                ConnectorType = ObjectStoreConnectorType.S3,
+                S3Endpoint = "http://localhost:9000",
+                // S3Bucket = "my-tidesdb",
+            },
+        };
+
+        Assert.Throws<ArgumentException>(() => TidesDb.Open(config));
+    }
+
+    [Fact]
+    public void ObjectStoreConfig_RequiresS3AccessKey()
+    {
+        var config = new Config
+        {
+            DbPath = _testDbPath,
+            ObjectStoreConfig = new ObjectStoreConfig
+            {
+                ConnectorType = ObjectStoreConnectorType.S3,
+                S3Endpoint = "http://localhost:9000",
+                S3Bucket = "my-tidesdb",
+                // S3AccessKey = "s3_access_key", 
+            },
+        };
+
+        Assert.Throws<ArgumentException>(() => TidesDb.Open(config));
+    }
+
+    [Fact]
+    public void ObjectStoreConfig_RequiresS3SecretKey()
+    {
+        var config = new Config
+        {
+            DbPath = _testDbPath,
+            ObjectStoreConfig = new ObjectStoreConfig
+            {
+                ConnectorType = ObjectStoreConnectorType.S3,
+                S3Endpoint = "http://localhost:9000",
+                S3Bucket = "my-tidesdb",
+                S3AccessKey = "s3_access_key", 
+                // S3SecretKey = "s3_secret_key",
+            },
+        };
+
+        Assert.Throws<ArgumentException>(() => TidesDb.Open(config));
+    }
+
     [Fact]
     public void OpenWithObjectStore_ReplicaMode_ShouldRejectWrites()
     {
@@ -1441,6 +1564,79 @@ public void OpenWithObjectStore_ReplicaMode_ShouldRejectWrites()
         }
     }
 
+    [Fact]
+    public void OpenWithObjectStore_S3_ReplicaMode_ShouldRejectWrites()
+    {
+        var objStoreDir = Path.Combine(Path.GetTempPath(), $"tidesdb_objstore_{Guid.NewGuid()}");
+        Directory.CreateDirectory(objStoreDir);
+
+        try
+        {
+            // First open as primary and create a CF
+            var primaryConfig = new Config
+            {
+                DbPath = _testDbPath,
+                NumFlushThreads = 1,
+                NumCompactionThreads = 1,
+                LogLevel = LogLevel.Info,
+                ObjectStoreConfig = new ObjectStoreConfig
+                {
+                    ConnectorType = ObjectStoreConnectorType.S3,
+                    S3Endpoint = "http://localhost:9000",
+                    S3Bucket = "my-tidesdb",
+                    S3AccessKey = "IoCMWHmye5ZjHoSwTIok",
+                    S3SecretKey = "YECnEaIco4s68zH3jqZ6HKlyC8FCZq5k1Ue4MLiq",
+                },
+            };
+
+            using (var primaryDb = TidesDb.Open(primaryConfig))
+            {
+                primaryDb.CreateColumnFamily("test_cf");
+                var cf = primaryDb.GetColumnFamily("test_cf")!;
+                using var txn = primaryDb.BeginTransaction();
+                txn.Put(cf, Encoding.UTF8.GetBytes("key1"), Encoding.UTF8.GetBytes("value1"));
+                txn.Commit();
+            }
+
+            // Open as replica
+            var replicaDbPath = Path.Combine(Path.GetTempPath(), $"tidesdb_replica_{Guid.NewGuid()}");
+            var replicaConfig = new Config
+            {
+                DbPath = replicaDbPath,
+                NumFlushThreads = 1,
+                NumCompactionThreads = 1,
+                LogLevel = LogLevel.Info,
+                ObjectStoreConfig = new ObjectStoreConfig
+                {
+                    ConnectorType = ObjectStoreConnectorType.S3,
+                    S3Endpoint = "http://localhost:9000",
+                    S3Bucket = "my-tidesdb",
+                    S3AccessKey = "IoCMWHmye5ZjHoSwTIok",
+                    S3SecretKey = "YECnEaIco4s68zH3jqZ6HKlyC8FCZq5k1Ue4MLiq",
+                    ReplicaMode = true,
+                    ReplicaSyncIntervalUs = 1_000_000,
+                },
+            };
+
+            try
+            {
+                using var replicaDb = TidesDb.Open(replicaConfig);
+                var dbStats = replicaDb.GetDbStats();
+                Assert.True(dbStats.ReplicaMode);
+            }
+            finally
+            {
+                if (Directory.Exists(replicaDbPath))
+                    Directory.Delete(replicaDbPath, true);
+            }
+        }
+        finally
+        {
+            if (Directory.Exists(objStoreDir))
+                Directory.Delete(objStoreDir, true);
+        }
+    }
+
     [Fact]
     public void TombstoneCfConfig_RoundTrip_ShouldPreserveValues()
     {