Skip to content

[Feature #1114] Implement active expiration for mutable region #1180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
d1defde
Active expiration WIP
hamdaankhalidmsft Apr 20, 2025
5cd822c
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft Apr 25, 2025
e01e10f
A lil more done
hamdaankhalidmsft Apr 25, 2025
09225b9
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft Apr 25, 2025
bb8dc08
WIP
hamdaankhalidmsft Apr 25, 2025
4a4230e
WIP
hamdaankhalidmsft Apr 25, 2025
63a17d2
Add server options
hamdaankhalidmsft Apr 30, 2025
76d7735
fix comment
hamdaankhalidmsft Apr 30, 2025
75de2d5
WIP
hamdaankhalidmsft Apr 30, 2025
ec8a0df
add default conf
hamdaankhalidmsft Apr 30, 2025
e583c5e
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalid Apr 30, 2025
ec97f47
fmt
hamdaankhalidmsft Apr 30, 2025
961f4a4
wip
hamdaankhalidmsft May 5, 2025
4ef899e
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft May 14, 2025
1caec60
Major WIP
hamdaankhalidmsft May 18, 2025
c50c97c
WIP
hamdaankhalidmsft May 18, 2025
3c06105
WIP
hamdaankhalidmsft May 19, 2025
e6ad222
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft May 23, 2025
3ba79bb
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft May 27, 2025
f41f90c
almost there buddy
hamdaankhalidmsft May 28, 2025
95f3441
no need for partial
hamdaankhalidmsft May 28, 2025
6384781
cleanup
hamdaankhalidmsft May 28, 2025
09e11a9
fmt
hamdaankhalidmsft May 28, 2025
eba834b
fmt
hamdaankhalidmsft May 28, 2025
4d15ded
Merge branch 'main' into hamdaankhalid/implement-active-expiration
hamdaankhalidmsft May 28, 2025
6662df3
Fix
hamdaankhalidmsft May 29, 2025
fba7f86
Add object store active expiration
hamdaankhalidmsft May 30, 2025
9fb4d70
active exp option
hamdaankhalidmsft May 30, 2025
e2623a2
fix object store iteration
hamdaankhalidmsft May 30, 2025
3cf851d
obj store and tests
hamdaankhalidmsft Jun 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions libs/host/Configuration/Options.cs
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,10 @@ public IEnumerable<string> LuaAllowedFunctions
[Option("max-databases", Required = false, HelpText = "Max number of logical databases allowed in a single Garnet server instance")]
public int MaxDatabases { get; set; }

[IntRangeValidation(-1, int.MaxValue, isRequired: false)]
[Option("active-expired-collection-freq", Required = false, HelpText = "expired key collection frequency in seconds")]
public int ActiveExpiredKeyCollectionFrequencySecs { get; set; }

/// <summary>
/// This property contains all arguments that were not parsed by the command line argument parser
/// </summary>
Expand Down Expand Up @@ -883,6 +887,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
UnixSocketPath = UnixSocketPath,
UnixSocketPermission = unixSocketPermissions,
MaxDatabases = MaxDatabases,
ActiveExpiredKeyCollectionFrequencySecs = ActiveExpiredKeyCollectionFrequencySecs,
};
}

Expand Down
5 changes: 4 additions & 1 deletion libs/host/defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -405,5 +405,8 @@
"UnixSocketPermission": 0,

/* Max number of logical databases allowed in a single Garnet server instance */
"MaxDatabases": 16
"MaxDatabases": 16,

/* Frequency of background expired key collection in seconds */
"ActiveExpiredKeyCollectionFrequencySecs": -1
}
120 changes: 116 additions & 4 deletions libs/server/Databases/DatabaseManagerBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
// Licensed under the MIT license.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -406,15 +408,15 @@ protected bool GrowIndexesIfNeeded(GarnetDatabase db)
/// <param name="logger">Logger</param>
protected void ExecuteObjectCollection(GarnetDatabase db, ILogger logger = null)
{
if (db.DatabaseStorageSession == null)
if (db.ObjectStoreCollectionDatabaseStorageSession == null)
{
var scratchBufferManager = new ScratchBufferManager();
db.DatabaseStorageSession =
db.ObjectStoreCollectionDatabaseStorageSession =
new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, Logger);
}

ExecuteHashCollect(db.DatabaseStorageSession);
ExecuteSortedSetCollect(db.DatabaseStorageSession);
ExecuteHashCollect(db.ObjectStoreCollectionDatabaseStorageSession);
ExecuteSortedSetCollect(db.ObjectStoreCollectionDatabaseStorageSession);
}

/// <summary>
Expand Down Expand Up @@ -691,5 +693,115 @@ private static void ExecuteSortedSetCollect(StorageSession storageSession)
storageSession.SortedSetCollect(ref storageSession.objectStoreBasicContext);
storageSession.scratchBufferManager.Reset();
}

/// <inheritdoc/>
public abstract void MainStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default);

/// <inheritdoc/>
public abstract void ObjStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default);

/// <inheritdoc/>
public abstract (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredMainStoreKeys(int dbId, ILogger logger = null);

/// <inheritdoc/>
public abstract (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredObjStoreKeys(int dbId, ILogger logger = null);

protected Task MainStoreCollectExpiredKeysForDbInBackgroundAsync(GarnetDatabase db, int frequency, ILogger logger = null, CancellationToken cancellationToken = default)
=> RunBackgroundTask(() => CollectExpiredMainStoreKeysImpl(db, logger), frequency, logger, cancellationToken);

protected Task ObjStoreCollectExpiredKeysForDbInBackgroundAsync(GarnetDatabase db, int frequency, ILogger logger = null, CancellationToken cancellationToken = default)
=> RunBackgroundTask(() => CollectExpiredObjStoreKeysImpl(db, logger), frequency, logger, cancellationToken);

private async Task RunBackgroundTask(Action action, int frequency, ILogger logger = null, CancellationToken cancellationToken = default)
{
Debug.Assert(frequency > 0);
try
{
while (true)
{
if (cancellationToken.IsCancellationRequested) return;
action();
await Task.Delay(TimeSpan.FromSeconds(frequency), cancellationToken);
}
}
catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
{ }
catch (Exception ex)
{
logger?.LogCritical(ex, "Unknown exception received for background task. Task won't be resumed.");
}
}


protected (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredMainStoreKeysImpl(GarnetDatabase db, ILogger logger = null)
{
if (db.MainStoreActiveExpDbStorageSession == null)
{
var scratchBufferManager = new ScratchBufferManager();
db.MainStoreActiveExpDbStorageSession =
new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, Logger);
}

long scanFrom = StoreWrapper.store.Log.ReadOnlyAddress;
long scanTill = StoreWrapper.store.Log.TailAddress;

(bool iteratedTillEndOfRange, long totalRecordsScanned) = db.MainStoreActiveExpDbStorageSession.ScanExpiredKeysMainStore(
cursor: scanFrom, storeCursor: out long scannedTill, keys: out List<byte[]> keys, endAddress: scanTill);

long numExpiredKeysFound = keys.Count;

RawStringInput input = new RawStringInput(RespCommand.DELIFEXPIM);
// If there is any sort of shift of the marker then a few of my scanned records will be from a redundant region.
// DelIfExpIm will be noop for those records since they will early exit at NCU.
foreach (byte[] key in keys)
{
unsafe
{
fixed (byte* keyPtr = key)
{
SpanByte keySb = SpanByte.FromPinnedPointer(keyPtr, key.Length);
// Use basic session for transient locking
db.MainStoreActiveExpDbStorageSession.DEL_Conditional(ref keySb, ref input, ref db.MainStoreActiveExpDbStorageSession.basicContext);
}
}

logger?.LogDebug("Deleted Expired Key {key} for DB {id}", System.Text.Encoding.UTF8.GetString(key), db.Id);
}

logger?.LogDebug("Main Store - Deleted {numKeys} keys out {totalRecords} records in range {start} to {end} for DB {id}", numExpiredKeysFound, totalRecordsScanned, scanFrom, scannedTill, db.Id);

return (numExpiredKeysFound, totalRecordsScanned);
}

protected (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredObjStoreKeysImpl(GarnetDatabase db, ILogger logger = null)
{
if (db.ObjStoreActiveExpDbStorageSession == null)
{
var scratchBufferManager = new ScratchBufferManager();
db.ObjStoreActiveExpDbStorageSession = new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, Logger);
}

long scanFrom = StoreWrapper.objectStore.Log.ReadOnlyAddress;
long scanTill = StoreWrapper.objectStore.Log.TailAddress;

(bool iteratedTillEndOfRange, long totalRecordsScanned) = db.ObjStoreActiveExpDbStorageSession.ScanExpiredKeysObjectStore(
cursor: scanFrom, storeCursor: out long scannedTill, keys: out List<byte[]> keys, endAddress: scanTill);

long numExpiredKeysFound = keys.Count;

ObjectInput input = new ObjectInput(new RespInputHeader(GarnetObjectType.DelIfExpIm));

for (var i = 0; i < keys.Count; i++)
{
var key = keys[i];
GarnetObjectStoreOutput output = new GarnetObjectStoreOutput();
db.ObjStoreActiveExpDbStorageSession.RMW_ObjectStore(ref key, ref input, ref output, ref db.ObjectStoreCollectionDatabaseStorageSession.objectStoreBasicContext);
logger?.LogDebug("Deleted Expired Key {key} for DB {id}", System.Text.Encoding.UTF8.GetString(key), db.Id);
}

logger?.LogDebug("Obj Store - Deleted {numKeys} keys out {totalRecords} records in range {start} to {end} for DB {id}", numExpiredKeysFound, totalRecordsScanned, scanFrom, scannedTill, db.Id);

return (numExpiredKeysFound, totalRecordsScanned);
}
}
}
22 changes: 22 additions & 0 deletions libs/server/Databases/IDatabaseManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -257,5 +257,27 @@ public Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, Cancellati
/// <param name="dbId">Database ID</param>
/// <returns>Functions state</returns>
internal FunctionsState CreateFunctionsState(int dbId = 0, byte respProtocolVersion = ServerOptions.DEFAULT_RESP_VERSION);

/// <summary>
/// Runs a background task per DB to manage periodic active expired key collection on Main store
/// In the case of a single DB, this will start only a single task.
/// </summary>
public void MainStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default);

/// <summary>
/// On Demand Expired Main Store Keys collection, for a db given its ID
/// </summary>
public (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredMainStoreKeys(int dbId, ILogger logger = null);

/// <summary>
/// Runs a background task per DB to manage periodic active expired key collection on Obj store
/// In the case of a single DB, this will start only a single task.
/// </summary>
public void ObjStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default);

/// <summary>
/// On Demand Expired Main Store Keys collection, for a db given its ID
/// </summary>
public (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredObjStoreKeys(int dbId, ILogger logger = null);
}
}
35 changes: 35 additions & 0 deletions libs/server/Databases/MultiDatabaseManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1039,5 +1039,40 @@ public override void Dispose()
databasesContentLock.CloseLock();
activeDbIds.mapLock.CloseLock();
}

public override void MainStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default)
=> StartCollectionPerDb((GarnetDatabase db) => MainStoreCollectExpiredKeysForDbInBackgroundAsync(db, frequency, logger, cancellation));

public override void ObjStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default)
=> StartCollectionPerDb((GarnetDatabase db) => MainStoreCollectExpiredKeysForDbInBackgroundAsync(db, frequency, logger, cancellation));

private void StartCollectionPerDb(Func<GarnetDatabase, Task> action)
{
var databasesMapSnapshot = databases.Map;

var activeDbIdsMapSize = activeDbIds.ActualSize;
var activeDbIdsMapSnapshot = activeDbIds.Map;

for (var i = 0; i < activeDbIdsMapSize; i++)
{
var dbId = activeDbIdsMapSnapshot[i];
GarnetDatabase db = databasesMapSnapshot[dbId];
Task.Run(() => action(db));
}
}

public override (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredMainStoreKeys(int dbId, ILogger logger = null)
=> CollectExpiredMainStoreKeysImpl(GetDbById(dbId), logger);

public override (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredObjStoreKeys(int dbId, ILogger logger = null)
=> CollectExpiredObjStoreKeysImpl(GetDbById(dbId), logger);

private GarnetDatabase GetDbById(int dbId)
{
var databasesMapSize = databases.ActualSize;
var databasesMapSnapshot = databases.Map;
Debug.Assert(dbId < databasesMapSize && databasesMapSnapshot[dbId] != null);
return databasesMapSnapshot[dbId];
}
}
}
18 changes: 18 additions & 0 deletions libs/server/Databases/SingleDatabaseManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,24 @@ private async Task<bool> TryPauseCheckpointsContinuousAsync(int dbId,
return checkpointsPaused;
}

public override (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredMainStoreKeys(int dbId, ILogger logger = null)
{
ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0);
return CollectExpiredMainStoreKeysImpl(DefaultDatabase, logger);
}

public override (long numExpiredKeysFound, long totalRecordsScanned) CollectExpiredObjStoreKeys(int dbId, ILogger logger = null)
{
ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0);
return CollectExpiredObjStoreKeysImpl(DefaultDatabase, logger);
}

public override void MainStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default)
=> Task.Run(() => MainStoreCollectExpiredKeysForDbInBackgroundAsync(DefaultDatabase, frequency, logger, cancellation));

public override void ObjStoreCollectExpiredKeysInBackgroundTask(int frequency, ILogger logger = null, CancellationToken cancellation = default)
=> Task.Run(() => MainStoreCollectExpiredKeysForDbInBackgroundAsync(DefaultDatabase, frequency, logger, cancellation));

private void SafeTruncateAOF(AofEntryType entryType, bool unsafeTruncateLog)
{
StoreWrapper.clusterProvider.SafeTruncateAOF(AppendOnlyFile.TailAddress);
Expand Down
16 changes: 14 additions & 2 deletions libs/server/GarnetDatabase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,17 @@ public class GarnetDatabase : IDisposable
/// <summary>
/// Storage session intended for store-wide object collection operations
/// </summary>
internal StorageSession DatabaseStorageSession;
internal StorageSession ObjectStoreCollectionDatabaseStorageSession;

/// <summary>
/// Storage session intended for main-store expired key collection operations
/// </summary>
internal StorageSession MainStoreActiveExpDbStorageSession;

/// <summary>
/// Storage session intended for object-store expired key collection operations
/// </summary>
internal StorageSession ObjStoreActiveExpDbStorageSession;

bool disposed = false;

Expand Down Expand Up @@ -165,7 +175,9 @@ public void Dispose()
ObjectStore?.Dispose();
AofDevice?.Dispose();
AppendOnlyFile?.Dispose();
DatabaseStorageSession?.Dispose();
ObjectStoreCollectionDatabaseStorageSession?.Dispose();
MainStoreActiveExpDbStorageSession?.Dispose();
ObjStoreActiveExpDbStorageSession?.Dispose();

if (ObjectStoreSizeTracker != null)
{
Expand Down
7 changes: 6 additions & 1 deletion libs/server/Objects/Types/GarnetObjectType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ public enum GarnetObjectType : byte

// Any new special type inserted here should update GarnetObjectTypeExtensions.FirstSpecialObjectType

/// <summary>
/// Special type indicating DELIFEXPIM commande, a conditional deletion when a key is in memory and expired
/// </summary>
DelIfExpIm = 0xf7,

/// <summary>
/// Special type indicating PEXPIRE command
/// </summary>
Expand Down Expand Up @@ -78,6 +83,6 @@ public static class GarnetObjectTypeExtensions
{
internal const GarnetObjectType LastObjectType = GarnetObjectType.Set;

internal const GarnetObjectType FirstSpecialObjectType = GarnetObjectType.PExpire;
internal const GarnetObjectType FirstSpecialObjectType = GarnetObjectType.DelIfExpIm;
}
}
48 changes: 48 additions & 0 deletions libs/server/Resp/AdminCommands.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ RespCommand.MIGRATE or
RespCommand.SLOWLOG_RESET => NetworkSlowLogReset(),
RespCommand.ROLE => NetworkROLE(),
RespCommand.SAVE => NetworkSAVE(),
RespCommand.ACTEXP => NetworkACTEXP(),
RespCommand.LASTSAVE => NetworkLASTSAVE(),
RespCommand.BGSAVE => NetworkBGSAVE(),
RespCommand.COMMITAOF => NetworkCOMMITAOF(),
Expand Down Expand Up @@ -891,6 +892,53 @@ private bool NetworkSAVE()
return true;
}

/// <summary>
/// ACTEXP MAIN|OBJ [DBID]
/// Scan the mutable region and tm=ombstone all expired keys actively instead of lazy.
/// This is meant to be able to let users do on-demand active expiration, and even build their own schedulers
/// for calling expiration based on their known workload patterns.
/// </summary>
private bool NetworkACTEXP()
{
if (parseState.Count < 1 || parseState.Count > 2)
{
return AbortWithWrongNumberOfArguments(nameof(RespCommand.ACTEXP));
}

StoreOptions storeOption;
if (!parseState.TryGetStoreOption(0, out storeOption))
{
while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_INVALID_STORE_OPTION, ref dcurr, dend))
SendAndReset();
return true;
}

// default database as default choice.
int dbId = 0;
if (parseState.Count > 1)
{
if (!TryParseDatabaseId(1, out dbId))
return true;
}

(long recordsExpired, long recordsScanned) = storeOption == StoreOptions.MAIN ?
storeWrapper.OnDemandMainStoreExpiredKeyCollection(dbId) : storeWrapper.OnDemandObjStoreExpiredKeyCollection(dbId);

// Resp Response Format => *2\r\n$NUM1\r\n$NUM2\r\n
int requiredSpace = 5 + NumUtils.CountDigits(recordsExpired) + 3 + NumUtils.CountDigits(recordsScanned) + 2;

while (!RespWriteUtils.TryWriteArrayLength(2, ref dcurr, dend))
SendAndReset();

while (!RespWriteUtils.TryWriteArrayItem(recordsExpired, ref dcurr, dend))
SendAndReset();

while (!RespWriteUtils.TryWriteArrayItem(recordsScanned, ref dcurr, dend))
SendAndReset();

return true;
}

/// <summary>
/// LASTSAVE [DBID]
/// </summary>
Expand Down
Loading
Loading