OAK-11634 : provided support for generations in FullGC (#2206)

* OAK-11634 : provided support for generations in FullGC

* OAK-11634 : avoid reset operation if full gc is not enabled

* OAK-11634 : changed log level to info

---------

Co-authored-by: Rishabh Kumar <diam@adobe.com>

Author: rishabhdaim

oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java

@@ -74,7 +74,8 @@ public static VersionGarbageCollector createVersionGC(final DocumentNodeStore no
                                                           boolean isFullGCDryRun, final DocumentNodeStoreBuilder<?> builder) {
         return new VersionGarbageCollector(nodeStore, gcSupport, isFullGCEnabled(builder), isFullGCDryRun,
                 isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
-                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
+                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis(),
+                builder.getFullGCGeneration());
     }
 
     public static DocumentNodeState readNode(DocumentNodeStore documentNodeStore, Path path, RevisionVector rootRevision) {

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java

@@ -36,6 +36,7 @@
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder.DEFAULT_UPDATE_LIMIT;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_ENABLED;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_GENERATION;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_PERFLOGGER_INFO_MILLIS;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
@@ -376,6 +377,17 @@
                     "is set to true, the fullGCMode will be ignored.")
     int fullGCMode() default DEFAULT_FULL_GC_MODE;
 
+    @AttributeDefinition(
+            name = "Document Node Store Full GC Generation",
+            description = "Long value indicating which Full GC generation is currently running on " +
+                    "document node store. The Default value is " + DEFAULT_FULL_GC_GENERATION +
+                    ". Note that this value can be overridden via framework " +
+                    "property 'oak.documentstore.fullGCGeneration'. " +
+                    "FullGC can be reset to run from beginning after incrementing this value. " +
+                    "Any value change must be a increment from previous value to reset the FullGC, " +
+                    "in case we set to a value smaller or equal to exiting generation, it would simply be ignored.")
+    long fullGCGeneration() default DEFAULT_FULL_GC_GENERATION;
+
     @AttributeDefinition(
             name = "Delay factor for a Full GC run",
             description = "A Full GC run has a gap of this delay factor to reduce continuous load on system." +

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java

@@ -653,7 +653,8 @@ public DocumentNodeStore(DocumentNodeStoreBuilder<?> builder) {
         this.versionGarbageCollector = new VersionGarbageCollector(
                 this, builder.createVersionGCSupport(), isFullGCEnabled(builder), false,
                 isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(),
-                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis());
+                builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis(),
+                builder.getFullGCGeneration());
         this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
         this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
         this.versionGarbageCollector.setFullGCPaths(builder.getFullGCIncludePaths(), builder.getFullGCExcludePaths());

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java

@@ -181,6 +181,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> {
     private Set<String> fullGCExcludePaths = Set.of();
     private boolean embeddedVerificationEnabled = DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
     private int fullGCMode = DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
+    private long fullGCGeneration = DocumentNodeStoreService.DEFAULT_FULL_GC_GENERATION;
     private long fullGcMaxAgeMillis = TimeUnit.SECONDS.toMillis(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE);
     private int fullGCBatchSize = DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
     private int fullGCProgressSize = DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
@@ -371,6 +372,15 @@ public int getFullGCMode() {
         return this.fullGCMode;
     }
 
+    public T setFullGCGeneration(long v) {
+        this.fullGCGeneration = v;
+        return thisBuilder();
+    }
+
+    public long getFullGCGeneration() {
+        return this.fullGCGeneration;
+    }
+
     /**
      * The maximum age for nodes in milliseconds. Older entries are candidates for full gc
      * @param v max age in millis

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java

@@ -141,6 +141,7 @@ public class DocumentNodeStoreService {
     static final boolean DEFAULT_FULL_GC_ENABLED = false;
     static final boolean DEFAULT_EMBEDDED_VERIFICATION_ENABLED = true;
     static final int DEFAULT_FULL_GC_MODE = 0;
+    static final int DEFAULT_FULL_GC_GENERATION = 0;
     static final int DEFAULT_MONGO_LEASE_SO_TIMEOUT_MILLIS = 30000;
     static final String DEFAULT_PERSISTENT_CACHE = "cache";
     static final String DEFAULT_JOURNAL_CACHE = "diff-cache";
@@ -530,6 +531,7 @@ private void configureBuilder(DocumentNodeStoreBuilder<?> builder) {
                 setFullGCExcludePaths(config.fullGCExcludePaths()).
                 setEmbeddedVerificationEnabled(config.embeddedVerificationEnabled()).
                 setFullGCMode(config.fullGCMode()).
+                setFullGCGeneration(config.fullGCGeneration()).
                 setFullGcMaxAgeMillis(TimeUnit.SECONDS.toMillis(config.fullGcMaxAgeInSecs())).
                 setFullGCBatchSize(config.fullGCBatchSize()).
                 setFullGCProgressSize(config.fullGCProgressSize()).

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java

@@ -83,11 +83,13 @@
 import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_GENERATION;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.BRANCH_COMMITS;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COLLISIONS;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COMMIT_ROOT;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.LOG;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
 import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.REVISIONS;
@@ -153,6 +155,11 @@ public class VersionGarbageCollector {
      */
     static final String SETTINGS_COLLECTION_FULL_GC_DRY_RUN_TIMESTAMP_PROP = "fullGCDryRunTimeStamp";
 
+    /**
+     * Property name to fullGcGeneration which is currently running
+     */
+    static final String SETTINGS_COLLECTION_FULL_GC_GENERATION_PROP = "fullGCGeneration";
+
     /**
      * Property name to _id till when last full-GC run happened in dryRun mode only
      */
@@ -174,6 +181,78 @@ static void setFullGcMode(int fullGcMode) {
         VersionGarbageCollector.fullGcMode = FullGCMode.getMode(fullGcMode);
     }
 
+    /**
+     * Sets the full GC generation for this document store and performs a reset if needed.
+     * <p>
+     * This method checks the existing full GC generation stored in the settings document:
+     * <ul>
+     *   <li>If no document exists, the new generation value is persisted</li>
+     *   <li>If the previous generation isn't a number or null, resets full GC and persists the new value</li>
+     *   <li>If the new generation is higher than the previous one, resets full GC and updates the value</li>
+     *   <li>If the new generation is less than or equal to the previous one, only logs the information</li>
+     * </ul>
+     *
+     * @param fullGcGen The new full GC generation value to set
+     * @return the generation with which the full GC has started
+     */
+    long resetFullGcIfGenChange(final long fullGcGen) {
+
+        if (fullGcGen == DEFAULT_FULL_GC_GENERATION) {
+            // generation hasn't been set yet, no need to make any change to make this backward compatible
+            LOG.info("Full GC generation is set to default value {}. No action needed.", fullGcGen);
+            return fullGcGen;
+        }
+
+        final Document doc = ds.find(SETTINGS, SETTINGS_COLLECTION_ID);
+
+        if (doc == null) {
+            // No version gc document exists, must be a new environment
+            persistFullGcGen(fullGcGen);
+            return fullGcGen;
+        }
+
+        final Object prevFullGcGenObj = doc.get(SETTINGS_COLLECTION_FULL_GC_GENERATION_PROP);
+
+        // If no previous generation or not a Number, just persist the new value
+        if (!(prevFullGcGenObj instanceof Number)) {
+            // this could happen if the previous value was set to a non-numeric value i.e. (not present)
+            LOG.info("Full GC generation {} is not a valid number or null, resetting to {}.", prevFullGcGenObj, fullGcGen);
+            resetFullGC();
+            persistFullGcGen(fullGcGen);
+            return fullGcGen;
+        }
+
+        // Compare with the previous generation
+        long prevFullGcGen = ((Number) prevFullGcGenObj).longValue();
+        if (prevFullGcGen >= fullGcGen) {
+            LOG.info("Full GC generation {} is less than or equal to the previously saved value {}.", fullGcGen, prevFullGcGen);
+            return prevFullGcGen;
+        } else {
+            LOG.info("Found a new generation of FullGC {}, resetting the Old gen {} values.", fullGcGen, prevFullGcGen);
+            resetFullGC();
+            persistFullGcGen(fullGcGen);
+            return fullGcGen;
+        }
+    }
+
+    /**
+     * Persists the full garbage collection generation value to the settings document.
+     * <p>
+     * This method creates or updates a document in the settings collection with the
+     * specified full GC generation number. The generation value is used to track major
+     * changes in the garbage collection process across restarts or different cluster nodes.
+     * <p>
+     * When the system detects a higher generation number than previously stored, it will
+     * reset the full GC state before persisting the new generation value.
+     *
+     * @param fullGcGeneration The full garbage collection generation value to persist
+     */
+    private void persistFullGcGen(long fullGcGeneration) {
+        UpdateOp op = new UpdateOp(SETTINGS_COLLECTION_ID, true);
+        op.set(SETTINGS_COLLECTION_FULL_GC_GENERATION_PROP, fullGcGeneration);
+        ds.createOrUpdate(SETTINGS, op);
+    }
+
     private final DocumentNodeStore nodeStore;
     private final DocumentStore ds;
     private final boolean fullGCEnabled;
@@ -198,7 +277,7 @@ static void setFullGcMode(int fullGcMode) {
                             final boolean isFullGCDryRun,
                             final boolean embeddedVerification) {
         this(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification, DEFAULT_FULL_GC_MODE,
-                0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE));
+                0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE), 0);
     }
 
     VersionGarbageCollector(DocumentNodeStore nodeStore,
@@ -210,7 +289,8 @@ static void setFullGcMode(int fullGcMode) {
                             final double fullGCDelayFactor,
                             final int fullGCBatchSize,
                             final int fullGCProgressSize,
-                            final long fullGcMaxAgeInMillis) {
+                            final long fullGcMaxAgeInMillis,
+                            final long fullGcGeneration) {
         this.nodeStore = nodeStore;
         this.versionStore = gcSupport;
         this.ds = gcSupport.getDocumentStore();
@@ -224,8 +304,9 @@ static void setFullGcMode(int fullGcMode) {
         this.options = new VersionGCOptions();
 
         setFullGcMode(fullGCMode);
-        AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, maxFullGcAgeInMillis: {}, batchSize: {}, progressSize: {}, delayFactor: {}",
-                fullGcMode, fullGcMaxAgeInMillis, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor);
+        long fullGcGen = fullGCEnabled ? resetFullGcIfGenChange(fullGcGeneration) : fullGcGeneration;
+        AUDIT_LOG.info("<init> VersionGarbageCollector created with fullGcMode: {}, maxFullGcAgeInMillis: {}, batchSize: {}, progressSize: {}, delayFactor: {}, fullGcGeneration: {}",
+                fullGcMode, fullGcMaxAgeInMillis, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor, fullGcGen);
     }
 
     /**

oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java

@@ -183,6 +183,19 @@ public int getFullGCMode() {
         return 0;
     }
 
+    @Override
+    public RDBDocumentNodeStoreBuilder setFullGCGeneration(long v) {
+        // fullGC modes are not supported for RDB
+        log.warn("FullGC generation are not supported for RDB");
+        return thisBuilder();
+    }
+
+    @Override
+    public long getFullGCGeneration() {
+        // fullGC modes are not supported for RDB
+        return 0;
+    }
+
     @Override
     public RDBDocumentNodeStoreBuilder setFullGcMaxAgeMillis(long v) {
         // fullGC modes are not supported for RDB

oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java

@@ -39,6 +39,7 @@
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_ENABLED;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_GENERATION;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE;
 import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_THROTTLING_ENABLED;
 import static org.junit.Assert.assertArrayEquals;
@@ -97,6 +98,7 @@ public void defaultValues() throws Exception {
         assertEquals(DEFAULT_THROTTLING_ENABLED, config.throttlingEnabled());
         assertEquals(DEFAULT_FULL_GC_ENABLED, config.fullGCEnabled());
         assertEquals(DEFAULT_FULL_GC_MODE, config.fullGCMode());
+        assertEquals(DEFAULT_FULL_GC_GENERATION, config.fullGCGeneration());
         assertEquals(DEFAULT_FGC_DELAY_FACTOR, config.fullGCDelayFactor(), 0.01);
         assertEquals(DEFAULT_FGC_BATCH_SIZE, config.fullGCBatchSize());
         assertEquals(DEFAULT_FGC_PROGRESS_SIZE, config.fullGCProgressSize());
@@ -139,6 +141,14 @@ public void fullGCModeValueSet() throws Exception {
         assertEquals(fullGCModeValue, config.fullGCMode());
     }
 
+    @Test
+    public void fullGCGenerationValueSet() throws Exception {
+        long fullGCGenerationValue = 2;
+        addConfigurationEntry(preset, "fullGCGeneration", fullGCGenerationValue);
+        Configuration config = createConfiguration();
+        assertEquals(fullGCGenerationValue, config.fullGCGeneration());
+    }
+
     @Test
     public void fullGCIncludePaths() throws Exception {
         final String[] includesPath = new String[]{"/foo", "/bar"};