OAK-11433 : added OSGi config for setting max revision age for full gc (#2032)

* OAK-11433 : added OSGi config for setting max revision age for full gc

* OAK-11433 : removed un-used var

---------

Co-authored-by: Rishabh Kumar <diam@adobe.com>

Author: rishabhdaim

oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java

@@ -74,7 +74,7 @@ public static VersionGarbageCollector createVersionGC(final DocumentNodeStore no
                                                           boolean isFullGCDryRun, final DocumentNodeStoreBuilder<?> builder) {
         return new VersionGarbageCollector(nodeStore, gcSupport, isFullGCEnabled(builder), isFullGCDryRun,
                 isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
-                builder.getFullGCBatchSize(), builder.getFullGCProgressSize());
+                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
     }
 
     public static DocumentNodeState readNode(DocumentNodeStore documentNodeStore, Path path, RevisionVector rootRevision) {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java

@@ -349,6 +349,14 @@
                     "property 'oak.documentstore.fullGCEnabled'")
     boolean fullGCEnabled() default DEFAULT_FULL_GC_ENABLED;
 
+    @AttributeDefinition(
+            name = "Full GC Max Age (in secs)",
+            description = "Version Garbage Collector (Full GC) logic will only consider those nodes for Full GC which " +
+                    "are not accessed recently (currentTime - lastModifiedTime > fullGcMaxAgeInSecs). For " +
+                    "example as per default only those document which have not been *updated* 24 hrs ago will be " +
+                    "considered for Full GC.")
+    long fullGcMaxAgeInSecs() default DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
+
     @AttributeDefinition(
             name = "Document Node Store Embedded Verification for Full GC",
             description = "Boolean value indicating whether Embedded Verification (i.e. verify the document after " +

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java

@@ -656,7 +656,7 @@ public DocumentNodeStore(DocumentNodeStoreBuilder<?> builder) {
         this.versionGarbageCollector = new VersionGarbageCollector(
                 this, builder.createVersionGCSupport(), isFullGCEnabled(builder), false,
                 isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
-                builder.getFullGCBatchSize(), builder.getFullGCProgressSize());
+                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
         this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
         this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
         this.versionGarbageCollector.setFullGCPaths(builder.getFullGCIncludePaths(), builder.getFullGCExcludePaths());

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java

@@ -181,6 +181,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
     private Set<String> fullGCExcludePaths = Set.of();
     private boolean embeddedVerificationEnabled = DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
     private int fullGCMode = DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
+    private long fullGcMaxAgeMillis = TimeUnit.SECONDS.toMillis(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE);
     private int fullGCBatchSize = DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
     private int fullGCProgressSize = DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
     private double fullGCDelayFactor = DocumentNodeStoreService.DEFAULT_FGC_DELAY_FACTOR;
@@ -360,6 +361,26 @@ public int getFullGCMode() {
         return this.fullGCMode;
     }
 
+    /**
+     * The maximum age for nodes in milliseconds. Older entries are candidates for full gc
+     * @param v max age in millis
+     * @return builder object
+     */
+    public T setFullGcMaxAgeMillis(long v) {
+        this.fullGcMaxAgeMillis = v;
+        return thisBuilder();
+    }
+
+    /**
+     * The maximum age for nodes in milliseconds. Older entries
+     * are candidates for Full GC.
+     *
+     * @return maximum age for nodes entries in milliseconds.
+     */
+    public long getFullGcMaxAgeMillis() {
+        return this.fullGcMaxAgeMillis;
+    }
+
     public T setFullGCBatchSize(int v) {
         this.fullGCBatchSize = v;
         return thisBuilder();

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java

@@ -165,6 +165,11 @@ public class DocumentNodeStoreService {
      */
     static final long DEFAULT_VER_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1);
 
+    /**
+     * Nodes older than this time would be garbage collected by Full GC
+     */
+    static final long DEFAULT_FULL_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1);
+
 
     /**
      * Blob modified before this time duration would be considered for Blob GC
@@ -523,6 +528,7 @@ private void configureBuilder(DocumentNodeStoreBuilder<?> builder) {
                 setFullGCExcludePaths(config.fullGCExcludePaths()).
                 setEmbeddedVerificationEnabled(config.embeddedVerificationEnabled()).
                 setFullGCMode(config.fullGCMode()).
+                setFullGcMaxAgeMillis(TimeUnit.SECONDS.toMillis(config.fullGcMaxAgeInSecs())).
                 setFullGCBatchSize(config.fullGCBatchSize()).
                 setFullGCProgressSize(config.fullGCProgressSize()).
                 setFullGCDelayFactor(config.fullGCDelayFactor()).

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java

@@ -104,10 +104,11 @@ public class VersionGCRecommendations {
      * @param gcMonitor          monitor class for messages
      * @param fullGCEnabled      whether fullGC is enabled or not
      * @param isFullGCDryRun     whether fullGC is running in dryRun mode or not
+     * @param fullGcMaxAgeMs the maximum age for revisions to be collected by fullGC
      */
     VersionGCRecommendations(long maxRevisionAgeMs, Checkpoints checkpoints, boolean checkpointCleanup, Clock clock,
                                     VersionGCSupport vgc, VersionGCOptions options, GCMonitor gcMonitor,
-                                    boolean fullGCEnabled, boolean isFullGCDryRun) {
+                                    boolean fullGCEnabled, boolean isFullGCDryRun, long fullGcMaxAgeMs) {
         boolean ignoreDueToCheckPoint;
         boolean ignoreFullGCDueToCheckPoint;
         long deletedOnceCount = 0;
@@ -185,9 +186,11 @@ public class VersionGCRecommendations {
             }
         }
 
+        TimeInterval keepFullGc = new TimeInterval(clock.getTime() - fullGcMaxAgeMs, Long.MAX_VALUE);
+
         TimeInterval scopeFullGC = new TimeInterval(isFullGCDryRun ? oldestModifiedDryRunDocTimeStamp.get() :
                 oldestModifiedDocTimeStamp.get(), MAX_VALUE);
-        scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs);
+        scopeFullGC = scopeFullGC.notLaterThan(keepFullGc.fromMs);
 
         suggestedIntervalMs = (long) settings.get(SETTINGS_COLLECTION_REC_INTERVAL_PROP);
         if (suggestedIntervalMs > 0) {
@@ -248,7 +251,7 @@ public class VersionGCRecommendations {
         this.scopeFullGC = scopeFullGC;
         this.fullGCId = isFullGCDryRun ? oldestModifiedDryRunDocId : oldestModifiedDocId;
         this.scopeIsComplete = scope.toMs >= keep.fromMs;
-        this.fullGCScopeIsComplete = scopeFullGC.toMs >= keep.fromMs;
+        this.fullGCScopeIsComplete = scopeFullGC.toMs >= keepFullGc.fromMs;
         this.maxCollect = collectLimit;
         this.suggestedIntervalMs = suggestedIntervalMs;
         this.deleteCandidateCount = deletedOnceCount;

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java

@@ -83,6 +83,7 @@
 import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.BRANCH_COMMITS;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COLLISIONS;
@@ -179,6 +180,7 @@ static void setFullGcMode(int fullGcMode) {
     private final boolean isFullGCDryRun;
     private final boolean embeddedVerification;
     private final double fullGCDelayFactor;
+    private long fullGcMaxAgeInMillis;
     private final int fullGCBatchSize;
     private final int fullGCProgressSize;
     private Set<String> fullGCIncludePaths = Collections.emptySet();
@@ -196,7 +198,7 @@ static void setFullGcMode(int fullGcMode) {
                             final boolean isFullGCDryRun,
                             final boolean embeddedVerification) {
         this(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification, DEFAULT_FULL_GC_MODE,
-                0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE);
+                0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
     }
 
     VersionGarbageCollector(DocumentNodeStore nodeStore,
@@ -207,21 +209,23 @@ static void setFullGcMode(int fullGcMode) {
                             final int fullGCMode,
                             final double fullGCDelayFactor,
                             final int fullGCBatchSize,
-                            final int fullGCProgressSize) {
+                            final int fullGCProgressSize,
+                            final long fullGcMaxAgeInMillis) {
         this.nodeStore = nodeStore;
         this.versionStore = gcSupport;
         this.ds = gcSupport.getDocumentStore();
         this.fullGCEnabled = fullGCEnabled;
         this.isFullGCDryRun = isFullGCDryRun;
         this.embeddedVerification = embeddedVerification;
         this.fullGCDelayFactor = fullGCDelayFactor;
+        this.fullGcMaxAgeInMillis = fullGcMaxAgeInMillis;
         this.fullGCBatchSize = Math.min(fullGCBatchSize, fullGCProgressSize);
         this.fullGCProgressSize = fullGCProgressSize;
         this.options = new VersionGCOptions();
 
         setFullGcMode(fullGCMode);
-        AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
-                fullGcMode, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
+        AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, maxFullGcAgeInMillis: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
+                fullGcMode, fullGcMaxAgeInMillis, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
     }
 
     /**
@@ -239,6 +243,10 @@ void setFullGCPaths(@NotNull Set<String> includes, @NotNull Set<String> excludes
         AUDIT_LOG.info("Full GC paths set to include: {} and exclude: {} in mode {}", includes, excludes, fullGcMode);
     }
 
+    void setFullGcMaxAge(final long fullGcMaxAge, final TimeUnit unit) {
+        this.fullGcMaxAgeInMillis = unit.toMillis(fullGcMaxAge);
+    }
+
     public void setStatisticsProvider(StatisticsProvider provider) {
         this.gcStats = new RevisionGCStats(provider);
         this.fullGCStats = new FullGCStatsCollectorImpl(provider);
@@ -353,7 +361,7 @@ public VersionGCInfo getInfo(long maxRevisionAge, TimeUnit unit)
         long now = nodeStore.getClock().getTime();
         VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
                 !nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled,
-                isFullGCDryRun);
+                isFullGCDryRun, fullGcMaxAgeInMillis);
         int estimatedIterations = -1;
         if (rec.suggestedIntervalMs > 0) {
             estimatedIterations = (int)Math.ceil((double) (now - rec.scope.toMs) / rec.suggestedIntervalMs);
@@ -734,7 +742,7 @@ private VersionGCStats gc(long maxRevisionAgeInMillis) throws IOException {
             stats.active.start();
             VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(),
                     !nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled,
-                    isFullGCDryRun);
+                    isFullGCDryRun, fullGcMaxAgeInMillis);
             GCPhases phases = new GCPhases(cancel, stats, gcMonitor);
             try {
                 if (!isFullGCDryRun) {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java

@@ -170,6 +170,19 @@ public int getFullGCMode() {
         // fullGC modes are not supported for RDB
         return 0;
     }
+
+    @Override
+    public RDBDocumentNodeStoreBuilder setFullGcMaxAgeMillis(long v) {
+        // fullGC modes are not supported for RDB
+        log.warn("FullGC Max Age is not supported for RDB");
+        return thisBuilder();
+    }
+
+    @Override
+    public long getFullGcMaxAgeMillis() {
+        // fullGC max age is not supported for RDB
+        return 0;
+    }
 
     @Override
     public RDBDocumentNodeStoreBuilder setDocStoreFullGCFeature(@Nullable Feature docStoreFullGC) {