diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java index b52c853ba55..454f73e47c4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java @@ -74,7 +74,7 @@ public static VersionGarbageCollector createVersionGC(final DocumentNodeStore no boolean isFullGCDryRun, final DocumentNodeStoreBuilder builder) { return new VersionGarbageCollector(nodeStore, gcSupport, isFullGCEnabled(builder), isFullGCDryRun, isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(), - builder.getFullGCBatchSize(), builder.getFullGCProgressSize()); + builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis()); } public static DocumentNodeState readNode(DocumentNodeStore documentNodeStore, Path path, RevisionVector rootRevision) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java index 6dec39e11b9..9e7921036aa 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Configuration.java @@ -349,6 +349,14 @@ "property 'oak.documentstore.fullGCEnabled'") boolean fullGCEnabled() default DEFAULT_FULL_GC_ENABLED; + @AttributeDefinition( + name = "Full GC Max Age (in secs)", + description = "Version Garbage Collector (Full GC) logic will only consider those nodes for Full GC which " + + "are not accessed recently (currentTime - lastModifiedTime > fullGcMaxAgeInSecs). For " + + "example as per default only those document which have not been *updated* 24 hrs ago will be " + + "considered for Full GC.") + long fullGcMaxAgeInSecs() default DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE; + @AttributeDefinition( name = "Document Node Store Embedded Verification for Full GC", description = "Boolean value indicating whether Embedded Verification (i.e. verify the document after " + diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index e056cb0f019..95bd69017ab 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -656,7 +656,7 @@ public DocumentNodeStore(DocumentNodeStoreBuilder builder) { this.versionGarbageCollector = new VersionGarbageCollector( this, builder.createVersionGCSupport(), isFullGCEnabled(builder), false, isEmbeddedVerificationEnabled(builder), builder.getFullGCMode(), builder.getFullGCDelayFactor(), - builder.getFullGCBatchSize(), builder.getFullGCProgressSize()); + builder.getFullGCBatchSize(), builder.getFullGCProgressSize(), builder.getFullGcMaxAgeMillis()); this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider()); this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor()); this.versionGarbageCollector.setFullGCPaths(builder.getFullGCIncludePaths(), builder.getFullGCExcludePaths()); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java index c260c82b8de..7b94796fe93 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java @@ -181,6 +181,7 @@ public class DocumentNodeStoreBuilder> { private Set fullGCExcludePaths = Set.of(); private boolean embeddedVerificationEnabled = DocumentNodeStoreService.DEFAULT_EMBEDDED_VERIFICATION_ENABLED; private int fullGCMode = DocumentNodeStoreService.DEFAULT_FULL_GC_MODE; + private long fullGcMaxAgeMillis = TimeUnit.SECONDS.toMillis(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE); private int fullGCBatchSize = DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE; private int fullGCProgressSize = DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE; private double fullGCDelayFactor = DocumentNodeStoreService.DEFAULT_FGC_DELAY_FACTOR; @@ -360,6 +361,26 @@ public int getFullGCMode() { return this.fullGCMode; } + /** + * The maximum age for nodes in milliseconds. Older entries are candidates for full gc + * @param v max age in millis + * @return builder object + */ + public T setFullGcMaxAgeMillis(long v) { + this.fullGcMaxAgeMillis = v; + return thisBuilder(); + } + + /** + * The maximum age for nodes in milliseconds. Older entries + * are candidates for Full GC. + * + * @return maximum age for nodes entries in milliseconds. + */ + public long getFullGcMaxAgeMillis() { + return this.fullGcMaxAgeMillis; + } + public T setFullGCBatchSize(int v) { this.fullGCBatchSize = v; return thisBuilder(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java index 31e79fb61ee..1437945d9a5 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java @@ -165,6 +165,11 @@ public class DocumentNodeStoreService { */ static final long DEFAULT_VER_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1); + /** + * Nodes older than this time would be garbage collected by Full GC + */ + static final long DEFAULT_FULL_GC_MAX_AGE = 24 * 60 * 60; //TimeUnit.DAYS.toSeconds(1); + /** * Blob modified before this time duration would be considered for Blob GC @@ -523,6 +528,7 @@ private void configureBuilder(DocumentNodeStoreBuilder builder) { setFullGCExcludePaths(config.fullGCExcludePaths()). setEmbeddedVerificationEnabled(config.embeddedVerificationEnabled()). setFullGCMode(config.fullGCMode()). + setFullGcMaxAgeMillis(TimeUnit.SECONDS.toMillis(config.fullGcMaxAgeInSecs())). setFullGCBatchSize(config.fullGCBatchSize()). setFullGCProgressSize(config.fullGCProgressSize()). setFullGCDelayFactor(config.fullGCDelayFactor()). diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java index 72afd946ab0..3d92e4e1448 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCRecommendations.java @@ -104,10 +104,11 @@ public class VersionGCRecommendations { * @param gcMonitor monitor class for messages * @param fullGCEnabled whether fullGC is enabled or not * @param isFullGCDryRun whether fullGC is running in dryRun mode or not + * @param fullGcMaxAgeMs the maximum age for revisions to be collected by fullGC */ VersionGCRecommendations(long maxRevisionAgeMs, Checkpoints checkpoints, boolean checkpointCleanup, Clock clock, VersionGCSupport vgc, VersionGCOptions options, GCMonitor gcMonitor, - boolean fullGCEnabled, boolean isFullGCDryRun) { + boolean fullGCEnabled, boolean isFullGCDryRun, long fullGcMaxAgeMs) { boolean ignoreDueToCheckPoint; boolean ignoreFullGCDueToCheckPoint; long deletedOnceCount = 0; @@ -185,9 +186,11 @@ public class VersionGCRecommendations { } } + TimeInterval keepFullGc = new TimeInterval(clock.getTime() - fullGcMaxAgeMs, Long.MAX_VALUE); + TimeInterval scopeFullGC = new TimeInterval(isFullGCDryRun ? oldestModifiedDryRunDocTimeStamp.get() : oldestModifiedDocTimeStamp.get(), MAX_VALUE); - scopeFullGC = scopeFullGC.notLaterThan(keep.fromMs); + scopeFullGC = scopeFullGC.notLaterThan(keepFullGc.fromMs); suggestedIntervalMs = (long) settings.get(SETTINGS_COLLECTION_REC_INTERVAL_PROP); if (suggestedIntervalMs > 0) { @@ -248,7 +251,7 @@ public class VersionGCRecommendations { this.scopeFullGC = scopeFullGC; this.fullGCId = isFullGCDryRun ? oldestModifiedDryRunDocId : oldestModifiedDocId; this.scopeIsComplete = scope.toMs >= keep.fromMs; - this.fullGCScopeIsComplete = scopeFullGC.toMs >= keep.fromMs; + this.fullGCScopeIsComplete = scopeFullGC.toMs >= keepFullGc.fromMs; this.maxCollect = collectLimit; this.suggestedIntervalMs = suggestedIntervalMs; this.deleteCandidateCount = deletedOnceCount; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java index 18c544a4953..c3ea9f75665 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java @@ -83,6 +83,7 @@ import static org.apache.jackrabbit.oak.plugins.document.Document.ID; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE; +import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MODE; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.BRANCH_COMMITS; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.COLLISIONS; @@ -237,6 +238,7 @@ static void setFullGcMode(int fullGcMode) { private final boolean isFullGCDryRun; private final boolean embeddedVerification; private final double fullGCDelayFactor; + private long fullGcMaxAgeInMillis; private final int fullGCBatchSize; private final int fullGCProgressSize; private Set fullGCIncludePaths = Collections.emptySet(); @@ -254,7 +256,7 @@ static void setFullGcMode(int fullGcMode) { final boolean isFullGCDryRun, final boolean embeddedVerification) { this(nodeStore, gcSupport, fullGCEnabled, isFullGCDryRun, embeddedVerification, DEFAULT_FULL_GC_MODE, - 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE); + 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); } VersionGarbageCollector(DocumentNodeStore nodeStore, @@ -265,7 +267,8 @@ static void setFullGcMode(int fullGcMode) { final int fullGCMode, final double fullGCDelayFactor, final int fullGCBatchSize, - final int fullGCProgressSize) { + final int fullGCProgressSize, + final long fullGcMaxAgeInMillis) { this.nodeStore = nodeStore; this.versionStore = gcSupport; this.ds = gcSupport.getDocumentStore(); @@ -273,13 +276,14 @@ static void setFullGcMode(int fullGcMode) { this.isFullGCDryRun = isFullGCDryRun; this.embeddedVerification = embeddedVerification; this.fullGCDelayFactor = fullGCDelayFactor; + this.fullGcMaxAgeInMillis = fullGcMaxAgeInMillis; this.fullGCBatchSize = Math.min(fullGCBatchSize, fullGCProgressSize); this.fullGCProgressSize = fullGCProgressSize; this.options = new VersionGCOptions(); setFullGcMode(fullGCMode); - AUDIT_LOG.info(" VersionGarbageCollector created with fullGcMode: {}, batchSize: {}, progressSize: {}, delayFactor: {}", - fullGcMode, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor); + AUDIT_LOG.info(" VersionGarbageCollector created with fullGcMode: {}, maxFullGcAgeInMillis: {}, batchSize: {}, progressSize: {}, delayFactor: {}", + fullGcMode, fullGcMaxAgeInMillis, fullGCBatchSize, fullGCProgressSize, fullGCDelayFactor); } /** @@ -297,6 +301,10 @@ void setFullGCPaths(@NotNull Set includes, @NotNull Set excludes AUDIT_LOG.info("Full GC paths set to include: {} and exclude: {} in mode {}", includes, excludes, fullGcMode); } + void setFullGcMaxAge(final long fullGcMaxAge, final TimeUnit unit) { + this.fullGcMaxAgeInMillis = unit.toMillis(fullGcMaxAge); + } + public void setStatisticsProvider(StatisticsProvider provider) { this.gcStats = new RevisionGCStats(provider); this.fullGCStats = new FullGCStatsCollectorImpl(provider); @@ -411,7 +419,7 @@ public VersionGCInfo getInfo(long maxRevisionAge, TimeUnit unit) long now = nodeStore.getClock().getTime(); VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(), !nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled, - isFullGCDryRun); + isFullGCDryRun, fullGcMaxAgeInMillis); int estimatedIterations = -1; if (rec.suggestedIntervalMs > 0) { estimatedIterations = (int)Math.ceil((double) (now - rec.scope.toMs) / rec.suggestedIntervalMs); @@ -792,7 +800,7 @@ private VersionGCStats gc(long maxRevisionAgeInMillis) throws IOException { stats.active.start(); VersionGCRecommendations rec = new VersionGCRecommendations(maxRevisionAgeInMillis, nodeStore.getCheckpoints(), !nodeStore.isReadOnlyMode(), nodeStore.getClock(), versionStore, options, gcMonitor, fullGCEnabled, - isFullGCDryRun); + isFullGCDryRun, fullGcMaxAgeInMillis); GCPhases phases = new GCPhases(cancel, stats, gcMonitor); try { if (!isFullGCDryRun) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java index 69ef254408b..18cfb61f609 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java @@ -170,6 +170,19 @@ public int getFullGCMode() { // fullGC modes are not supported for RDB return 0; } + + @Override + public RDBDocumentNodeStoreBuilder setFullGcMaxAgeMillis(long v) { + // fullGC modes are not supported for RDB + log.warn("FullGC Max Age is not supported for RDB"); + return thisBuilder(); + } + + @Override + public long getFullGcMaxAgeMillis() { + // fullGC max age is not supported for RDB + return 0; + } @Override public RDBDocumentNodeStoreBuilder setDocStoreFullGCFeature(@Nullable Feature docStoreFullGC) { diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java index fc8921048e3..6bc5ed5c753 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java @@ -21,6 +21,7 @@ import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.build; +import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc; import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.allOrphProp; import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsEqual; import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsZero; @@ -145,7 +146,7 @@ public void unmergedAddChildren() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc,1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(), @@ -160,7 +161,7 @@ public void unmergedAddChildren() throws Exception { // now do another gc - should not have anything left to clean up though clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); - stats = gc.gc(1, HOURS); + stats = gc(gc,1, HOURS); assertStatsCountsZero(stats); if (!isModeOneOf(FullGCMode.NONE, FullGCMode.GAP_ORPHANS, FullGCMode.GAP_ORPHANS_EMPTYPROPS, FullGCMode.EMPTYPROPS)) { assertNotExists("1:/a"); @@ -204,7 +205,7 @@ public void unmergedAddThenMergedAddAndRemoveChildren() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); if (collisionsBeforeGC == 1) { // expects a collision to have happened - which was cleaned up - hence a _bc (but not the _revision I guess) @@ -241,7 +242,7 @@ public void unmergedAddThenMergedAddAndRemoveChildren() throws Exception { assertNotExists("1:/b"); clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertEquals(0, stats.updatedFullGCDocsCount); assertEquals(0, stats.deletedDocGCCount); assertEquals(0, stats.deletedUnmergedBCCount); @@ -288,7 +289,7 @@ public void testDeletedPropsAndUnmergedBC() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); // 6 deleted props: 0:/[_collisions], 1:/foo[p, a], 1:/bar[_bc,prop,_revisions] assertStatsCountsEqual(stats, @@ -331,7 +332,7 @@ public void unmergedAddTwoChildren() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(), @@ -351,7 +352,7 @@ public void unmergedAddTwoChildren() throws Exception { // now do another gc - should not have anything left to clean up though clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsZero(stats); if (!isModeOneOf(FullGCMode.NONE)) { assertBranchRevisionRemovedFromAllDocuments(store, br1); @@ -387,7 +388,7 @@ public void unmergedAddsThenMergedAddsChildren() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(), @@ -404,7 +405,7 @@ public void unmergedAddsThenMergedAddsChildren() throws Exception { // now do another gc to get those documents actually deleted clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsZero(stats); if (!isModeOneOf(FullGCMode.NONE)) { assertBranchRevisionRemovedFromAllDocuments(store, br1); @@ -452,7 +453,7 @@ public void unmergedAddsThenMergedAddThenUnmergedRemovesChildren() throws Except // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, empPropOnly(0, 0, 0, 0, 0, 0, 0), @@ -469,7 +470,7 @@ public void unmergedAddsThenMergedAddThenUnmergedRemovesChildren() throws Except // now do another gc to get those documents actually deleted clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsZero(stats); if (!isModeOneOf(FullGCMode.NONE)) { assertBranchRevisionRemovedFromAllDocuments(store, br1); @@ -494,7 +495,7 @@ public void unmergedAddAndRemoveChild() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS); + VersionGarbageCollector.VersionGCStats stats = gc(gc, 1, HOURS); // first gc round now deletes it, via orphaned node deletion assertStatsCountsEqual(stats, @@ -511,7 +512,7 @@ public void unmergedAddAndRemoveChild() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // now do second gc round - should not have anything left to clean up though - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsZero(stats); if (!isModeOneOf(FullGCMode.NONE)) { assertBranchRevisionRemovedFromAllDocuments(store, br); @@ -524,7 +525,7 @@ public void unmergedRemoveProperty() throws Exception { mergedBranchCommit(b -> b.child("foo").setProperty("a", "b")); // do a gc without waiting, to check that works fine store.runBackgroundOperations(); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertEquals(0, stats.updatedFullGCDocsCount); RevisionVector br = unmergedBranchCommit(b -> b.child("foo").removeProperty("a")); @@ -534,7 +535,7 @@ public void unmergedRemoveProperty() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(), @@ -560,7 +561,7 @@ public void unmergedAddProperty() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); // 1 deleted prop: 1:/foo[a] assertStatsCountsEqual(stats, @@ -589,7 +590,7 @@ public void unmergedRemoveChild() throws Exception { }); // do a gc without waiting, to check that works fine store.runBackgroundOperations(); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertEquals(0, stats.updatedFullGCDocsCount); assertEquals(0, stats.deletedUnmergedBCCount); @@ -605,7 +606,7 @@ public void unmergedRemoveChild() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(0, 0, 0, 0, 0, 0, 0), @@ -638,7 +639,7 @@ public void unmergedMergedRemoveChild() throws Exception { }); // do a gc without waiting, to check that works fine store.runBackgroundOperations(); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertEquals(0, stats.updatedFullGCDocsCount); assertEquals(0, stats.deletedUnmergedBCCount); @@ -655,7 +656,7 @@ public void unmergedMergedRemoveChild() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertStatsCountsEqual(stats, gapOrphOnly(0, 0, 0, 0, 0, 0, 0), @@ -681,7 +682,7 @@ public void unmergedThenMergedRemoveProperty() throws Exception { mergedBranchCommit(b -> b.child("foo").setProperty("c", "d")); // do a gc without waiting, to check that works fine store.runBackgroundOperations(); - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); assertEquals(0, stats.updatedFullGCDocsCount); assertEquals(0, stats.deletedUnmergedBCCount); assertEquals(0, stats.deletedPropsCount); @@ -712,7 +713,7 @@ public void unmergedThenMergedRemoveProperty() throws Exception { // wait two hours clock.waitUntil(clock.getTime() + HOURS.toMillis(2)); // clean everything older than one hour - stats = gc.gc(1, HOURS); + stats = gc(gc, 1, HOURS); // deleted props: 0:/[rootProp], 1:/foo[a] // deleted internal prop : 0:/ _collision diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java index f7131484207..398f6501748 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceConfigurationTest.java @@ -101,6 +101,7 @@ public void defaultValues() throws Exception { assertEquals(DEFAULT_FGC_PROGRESS_SIZE, config.fullGCProgressSize()); assertEquals(DEFAULT_FULL_GC_ENABLED, config.fullGCEnabled()); assertEquals(DEFAULT_EMBEDDED_VERIFICATION_ENABLED, config.embeddedVerificationEnabled()); + assertEquals(DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE, config.fullGcMaxAgeInSecs()); assertEquals(CommitQueue.DEFAULT_SUSPEND_TIMEOUT, config.suspendTimeoutMillis()); } @@ -184,6 +185,14 @@ public void fullGCDelayFactor() throws Exception { assertEquals(fullGCDelayFactor, config.fullGCDelayFactor(), 0.01); } + @Test + public void fullGcMaxAgeInSecs() throws Exception { + long fullGcMaxAgeInSecs = 30 * 24 * 60 * 60; // 30 days + addConfigurationEntry(preset, "fullGcMaxAgeInSecs", fullGcMaxAgeInSecs); + Configuration config = createConfiguration(); + assertEquals(fullGcMaxAgeInSecs, config.fullGcMaxAgeInSecs()); + } + @Test public void presetSocketKeepAlive() throws Exception { boolean keepAlive = !DocumentNodeStoreService.DEFAULT_SO_KEEP_ALIVE; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java index f75f180fe4b..6af0fbc4dbc 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/FullGCHelper.java @@ -26,12 +26,15 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import java.io.IOException; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import static org.apache.commons.lang3.reflect.FieldUtils.writeField; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -60,6 +63,16 @@ public static void disableFullGCDryRun(final VersionGarbageCollector vgc) throws } } + public static VersionGarbageCollector.VersionGCStats gc(final VersionGarbageCollector gc, final long maxRevisionAge, final TimeUnit unit) throws IOException { + gc.setFullGcMaxAge(maxRevisionAge, unit); // set full gc max age + final VersionGarbageCollector.VersionGCStats stats = gc.gc(maxRevisionAge, unit); + if (stats.skippedFullGCDocsCount != 0) { + (new Exception("here: " + stats.skippedFullGCDocsCount)).printStackTrace(System.out); + } + assertEquals(0, stats.skippedFullGCDocsCount); + return stats; + } + public static RevisionVector mergedBranchCommit(final NodeStore store, final Consumer buildFunction) throws Exception { return build(store, true, true, buildFunction); } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java index aff4a6b324d..55cf7642438 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCDeletionTest.java @@ -21,6 +21,7 @@ import static java.util.concurrent.Executors.newSingleThreadExecutor; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -109,14 +110,14 @@ public void deleteParentLast() throws Exception{ //sorted again by VersionGC and then /x would always come after /x/y try { ts.throwException = true; - gc.gc(maxAge * 2, HOURS); + gc(gc, maxAge * 2, HOURS); fail("Exception should be thrown"); } catch (AssertionError ignore) { } ts.throwException = false; - gc.gc(maxAge * 2, HOURS); + gc(gc, maxAge * 2, HOURS); assertNull(ts.find(Collection.NODES, "2:/x/y")); assertNull(ts.find(Collection.NODES, "1:/x")); } @@ -162,7 +163,7 @@ public void leaveResurrectedNodesAlone() throws Exception{ clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge * 2) + delta); VersionGarbageCollector gc = store.getVersionGarbageCollector(); - VersionGCStats stats = gc.gc(maxAge * 2, HOURS); + VersionGCStats stats = gc(gc, maxAge * 2, HOURS); assertEquals(1, stats.updateResurrectedGCCount); NodeDocument d4 = ts.find(Collection.NODES, id, 0); assertNotNull(d4); @@ -202,7 +203,7 @@ public void deleteLargeNumber() throws Exception{ VersionGarbageCollector gc = store.getVersionGarbageCollector(); gc.setOptions(gc.getOptions().withOverflowToDiskThreshold(100)); - VersionGCStats stats = gc.gc(maxAge * 2, HOURS); + VersionGCStats stats = gc(gc, maxAge * 2, HOURS); assertEquals(noOfDocsToDelete * 2 + 1, stats.deletedDocGCCount); assertEquals(noOfDocsToDelete, stats.deletedLeafDocGCCount); @@ -249,7 +250,7 @@ public void gcWithPathsHavingNewLine() throws Exception{ VersionGarbageCollector gc = store.getVersionGarbageCollector(); gc.setOptions(gc.getOptions().withOverflowToDiskThreshold(100)); - VersionGCStats stats = gc.gc(maxAge * 2, HOURS); + VersionGCStats stats = gc(gc, maxAge * 2, HOURS); assertEquals(noOfDocsToDelete * 2 + 1, stats.deletedDocGCCount); assertEquals(noOfDocsToDelete, stats.deletedLeafDocGCCount); } @@ -297,7 +298,7 @@ public void gcForPreviousDocs() throws Exception{ //Pass some time and run GC clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge * 2) + delta); VersionGarbageCollector gc = store.getVersionGarbageCollector(); - VersionGCStats stats = gc.gc(maxAge * 2, HOURS); + VersionGCStats stats = gc(gc, maxAge * 2, HOURS); //Asset GC stats assertEquals(2, stats.deletedDocGCCount); @@ -382,7 +383,7 @@ public List call() { // run GC once the reader thread is collecting documents ready.await(); VersionGarbageCollector gc = store.getVersionGarbageCollector(); - VersionGCStats stats = gc.gc(30, MINUTES); + VersionGCStats stats = gc(gc, 30, MINUTES); assertEquals(90, stats.deletedDocGCCount); assertEquals(90, stats.deletedLeafDocGCCount); diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java index edb4bfd6929..1b87fd5d468 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCTest.java @@ -55,6 +55,7 @@ import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE; +import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGC; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.disableFullGCDryRun; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC; @@ -136,7 +137,7 @@ public void failParallelGC() throws Exception { assertTrue(gcBlocked); // now try to trigger another GC try { - gc.gc(30, TimeUnit.MINUTES); + FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); fail("must throw an IOException"); } catch (IOException e) { assertTrue(e.getMessage().contains("already running")); @@ -209,7 +210,7 @@ public void cancelMustNotUpdateLastOldestModifiedTimeStamp() throws Exception { String versionGCId = SETTINGS_COLLECTION_ID; String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP; enableFullGC(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); Document statusBefore = store.find(SETTINGS, versionGCId); // block gc call store.semaphore.acquireUninterruptibly(); @@ -247,7 +248,7 @@ public void cancelMustNotUpdateLastOldestModifiedDocId() throws Exception { String versionGCId = SETTINGS_COLLECTION_ID; String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP; enableFullGC(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); Document statusBefore = store.find(SETTINGS, versionGCId); // block gc call store.semaphore.acquireUninterruptibly(); @@ -288,12 +289,12 @@ public void dryRunMustNotUpdateLastOldestModifiedTimeStamp() throws Exception { String versionGCId = SETTINGS_COLLECTION_ID; String fullGCTimestamp = SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP; enableFullGC(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); Document statusBefore = store.find(SETTINGS, versionGCId); // now run GC in dryRun mode enableFullGCDryRun(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); // ensure a dryRun GC doesn't update that versionGC SETTINGS entries Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID); @@ -309,11 +310,11 @@ public void dryRunMustNotUpdateLastOldestModifiedDocId() throws Exception { String versionGCId = SETTINGS_COLLECTION_ID; String oldestModifiedDocId = SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP; enableFullGC(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); final Document statusBefore = store.find(SETTINGS, versionGCId); // now run GC in dryRun mode enableFullGCDryRun(gc); - gc.gc(30, SECONDS); + FullGCHelper.gc(gc, 30, SECONDS); // ensure a dryRun GC doesn't update that versionGC SETTINGS entry final Document statusAfter = store.find(SETTINGS, SETTINGS_COLLECTION_ID); assertNotNull(statusAfter); @@ -326,7 +327,7 @@ public void dryRunMustNotUpdateLastOldestModifiedDocId() throws Exception { @Test public void getInfo() throws Exception { - gc.gc(1, TimeUnit.HOURS); + FullGCHelper.gc(gc, 1, TimeUnit.HOURS); gc.getInfo(1, TimeUnit.HOURS); } @@ -336,7 +337,7 @@ public void gcMonitorStatusUpdates() throws Exception { TestGCMonitor monitor = new TestGCMonitor(); gc.setGCMonitor(monitor); - gc.gc(30, TimeUnit.MINUTES); + FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); List expected = List.of("INITIALIZING", "COLLECTING", "CHECKING", "COLLECTING", "DELETING", "SORTING", @@ -349,7 +350,7 @@ public void gcMonitorInfoMessages() throws Exception { TestGCMonitor monitor = new TestGCMonitor(); gc.setGCMonitor(monitor); - gc.gc(2, TimeUnit.HOURS); + FullGCHelper.gc(gc, 2, TimeUnit.HOURS); List infoMessages = monitor.getInfoMessages(); assertEquals(3, infoMessages.size()); @@ -361,7 +362,7 @@ public void gcMonitorInfoMessages() throws Exception { @Test public void findVersionGC() throws Exception { store.findVersionGC.set(0); - gc.gc(1, TimeUnit.HOURS); + FullGCHelper.gc(gc, 1, TimeUnit.HOURS); // must only read once assertEquals(1, store.findVersionGC.get()); } @@ -377,7 +378,7 @@ public void recommendationsOnHugeBacklog() throws Exception { VersionGCSupport localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oneYearAgo, twelveTimesTheLimit); VersionGCRecommendations rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), - localgcsupport, options, new TestGCMonitor(), false, false); + localgcsupport, options, new TestGCMonitor(), false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); // should select a duration of roughly one month long duration= rec.scope.getDurationMs(); @@ -391,7 +392,7 @@ public void recommendationsOnHugeBacklog() throws Exception { assertTrue(stats.needRepeat); rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport, - options, new TestGCMonitor(), false, false); + options, new TestGCMonitor(), false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); // new duration should be half long nduration = rec.scope.getDurationMs(); @@ -420,7 +421,7 @@ public void expandIntervalAgain() throws Exception { // loop until the recommended interval is at 60s (precisionMS) do { rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport, - options, testmonitor, false, false); + options, testmonitor, false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); stats = new VersionGCStats(); stats.limitExceeded = true; rec.evaluate(stats); @@ -437,7 +438,7 @@ public void expandIntervalAgain() throws Exception { deletedCount -= deleted; localgcsupport = fakeVersionGCSupport(ns.getDocumentStore(), oldestDeleted, deletedCount); rec = new VersionGCRecommendations(secondsPerDay, ns.getCheckpoints(), true, ns.getClock(), localgcsupport, - options, testmonitor, false, false); + options, testmonitor, false, false, SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); stats = new VersionGCStats(); stats.limitExceeded = false; stats.deletedDocGCCount = deleted; @@ -464,7 +465,7 @@ public long getDeletedOnceCount() { }, false, false, false); // run first RGC - gc.gc(1, TimeUnit.HOURS); + FullGCHelper.gc(gc, 1, TimeUnit.HOURS); // afterwards there should be no more calls to getDeletedOnceCount() deletedOnceCountCalls.set(0); @@ -472,7 +473,7 @@ public long getDeletedOnceCount() { for (int i = 0; i < 10; i++) { advanceClock(5, SECONDS); - gc.gc(1, TimeUnit.HOURS); + FullGCHelper.gc(gc, 1, TimeUnit.HOURS); assertEquals(0, deletedOnceCountCalls.get()); } } @@ -481,7 +482,7 @@ public long getDeletedOnceCount() { @Test public void testFullGCDocumentRead_disabled() throws Exception { disableFullGC(gc); - VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES); + VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); assertNotNull(stats); assertEquals(0, stats.fullGCDocsElapsed); } @@ -489,7 +490,8 @@ public void testFullGCDocumentRead_disabled() throws Exception { @Test public void testFullGCDocumentRead_enabled() throws Exception { enableFullGC(gc); - VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES); + gc.setFullGcMaxAge(30, MINUTES); + VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); assertNotNull(stats); assertNotEquals(0, stats.fullGCDocsElapsed); } @@ -501,7 +503,7 @@ public void testFullGCDocumentRead_enabled() throws Exception { public void testFullGCDryRunModeEnabled() throws Exception { enableFullGC(gc); enableFullGCDryRun(gc); - VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES); + VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); assertNotNull(stats); assertTrue(stats.fullGCDryRunMode); } @@ -510,7 +512,7 @@ public void testFullGCDryRunModeEnabled() throws Exception { public void testResetFullGCDryRunMode() throws Exception { enableFullGC(gc); enableFullGCDryRun(gc); - VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES); + VersionGCStats stats = FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); assertNotNull(stats); // add dryRun fields data @@ -539,7 +541,7 @@ public void testResetFullGCDryRunMode() throws Exception { public void testVGCWithBatchSizeSmallerThanProgressSize() throws IllegalAccessException { VersionGarbageCollector vgc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - 0, 0, 1000, 5000); + 0, 0, 1000, 5000, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals(1000, readDeclaredField(vgc, "fullGCBatchSize", true)); assertEquals(5000, readDeclaredField(vgc, "fullGCProgressSize", true)); @@ -549,7 +551,7 @@ ns, new VersionGCSupport(store), true, false, false, public void testVGCWithBatchSizeGreaterThanProgressSize() throws IllegalAccessException { VersionGarbageCollector vgc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - 0, 0, 20000, 15000); + 0, 0, 20000, 15000, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals(15000, readDeclaredField(vgc, "fullGCBatchSize", true)); assertEquals(15000, readDeclaredField(vgc, "fullGCProgressSize", true)); @@ -570,7 +572,7 @@ public void testVersionGCLoadGCModeConfigurationNotApplicable() { // reinitialize VersionGarbageCollector with not allowed value VersionGarbageCollector gc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - fullGcModeNotAllowedValue, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE); + fullGcModeNotAllowedValue, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals("Starting VersionGarbageCollector with not applicable / not allowed value" + "will set fullGcMode to default NONE", VersionGarbageCollector.FullGCMode.NONE, VersionGarbageCollector.getFullGcMode()); @@ -581,7 +583,7 @@ public void testVersionGCLoadGCModeConfigurationNone() { int fullGcModeNone = 0; VersionGarbageCollector gc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - fullGcModeNone, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE); + fullGcModeNone, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals(VersionGarbageCollector.FullGCMode.NONE, VersionGarbageCollector.getFullGcMode()); } @@ -591,7 +593,7 @@ public void testVersionGCLoadGCModeConfigurationGapOrphans() { int fullGcModeGapOrphans = 2; VersionGarbageCollector gc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - fullGcModeGapOrphans, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE); + fullGcModeGapOrphans, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals(VersionGarbageCollector.FullGCMode.GAP_ORPHANS, VersionGarbageCollector.getFullGcMode()); } @@ -601,7 +603,7 @@ public void testVersionGCLoadGCModeConfigurationGapOrphansEmptyProperties() { int fullGcModeGapOrphansEmptyProperties = 3; VersionGarbageCollector gc = new VersionGarbageCollector( ns, new VersionGCSupport(store), true, false, false, - fullGcModeGapOrphansEmptyProperties, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE); + fullGcModeGapOrphansEmptyProperties, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE)); assertEquals(VersionGarbageCollector.FullGCMode.GAP_ORPHANS_EMPTYPROPS, VersionGarbageCollector.getFullGcMode()); } @@ -613,7 +615,7 @@ private Future gc() { return execService.submit(new Callable() { @Override public VersionGCStats call() throws Exception { - return gc.gc(30, TimeUnit.MINUTES); + return FullGCHelper.gc(gc, 30, TimeUnit.MINUTES); } }); } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java index 08aaea9cac8..a2ad70d6914 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java @@ -64,10 +64,12 @@ import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_BATCH_SIZE; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FGC_PROGRESS_SIZE; +import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_FULL_GC_MAX_AGE; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionNotRemovedFromAllDocuments; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.assertBranchRevisionRemovedFromAllDocuments; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGC; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.enableFullGCDryRun; +import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.gc; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.mergedBranchCommit; import static org.apache.jackrabbit.oak.plugins.document.FullGCHelper.unmergedBranchCommit; import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE; @@ -461,15 +463,6 @@ public void gcLongPathSplitDocs() throws Exception { gcSplitDocsInternal("sub".repeat(120)); } - private VersionGCStats gc(VersionGarbageCollector gc, long maxRevisionAge, TimeUnit unit) throws IOException { - final VersionGCStats stats = gc.gc(maxRevisionAge, unit); - if (stats.skippedFullGCDocsCount != 0) { - (new Exception("here: " + stats.skippedFullGCDocsCount)).printStackTrace(System.out); - } - assertEquals(0, stats.skippedFullGCDocsCount); - return stats; - } - // OAK-10199 @Test public void testFullGCNeedRepeat() throws Exception { @@ -588,6 +581,43 @@ public void testFullGCNotIgnoredForRGCCheckpoint() throws Exception { assertTrue(stats.canceled); } + // OAK-11433 + @Test + public void testFullGCNotIgnoredButRevisionGCIgnoredForCheckpoint() throws Exception { + long expiryTime = 100, maxRevisionGCAge = 20; + // enable the full gc flag + writeField(gc, "fullGCEnabled", true, true); + + // create a bunch of garbage + NodeBuilder b1 = store1.getRoot().builder(); + for (int i = 0; i < 100; i++) { + b1.child("c" + i).setProperty("test", "t", STRING); + } + store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY); + for (int i = 0; i < 100; i++) { + b1.child("c" + i).removeProperty("test"); + } + store1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY); + store1.runBackgroundOperations(); + + // create a checkpoint 1 hour in the past + String checkpoint = store1.checkpoint(TimeUnit.HOURS.toMillis(1)); + + // Fast forward time to future such that we are past the checkpoint + clock.waitUntil(clock.getTime() + expiryTime); + gc.setFullGcMaxAge(2, HOURS); + VersionGCStats stats = gc.gc(maxRevisionGCAge, TimeUnit.MILLISECONDS); + + // FullGC should not be ignored + assertFalse("Full GC should be performed", stats.ignoredFullGCDueToCheckPoint); + // RevisionGC should be ignored + assertTrue("Revision GC should be ignored due to checkpoint", stats.ignoredGCDueToCheckPoint); + assertFalse(stats.canceled); + assertFalse(stats.needRepeat); + } + + // OAK-11433 - END + @Test public void testGCDeletedLongPathPropsInclExcl_excludes() throws Exception { String longName = "p".repeat(PATH_LONG + 1); @@ -1691,7 +1721,7 @@ private Iterator candidates(long fromModified, long toModified, in //3. Check that deleted property does get collected post maxAge clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta); - stats = gcRef.get().gc(maxAge*2, HOURS); + stats = gc(gcRef.get(), maxAge*2, HOURS); assertTrue(stats.canceled); assertEquals(0, stats.updatedFullGCDocsCount); assertEquals(0, stats.deletedPropsCount); @@ -1759,7 +1789,7 @@ public Iterable getModifiedDocs(long fromModified, long toModified } }; - gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false, false, 3, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE)); + gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false, false, 3, 0, DEFAULT_FGC_BATCH_SIZE, DEFAULT_FGC_PROGRESS_SIZE, TimeUnit.SECONDS.toMillis(DEFAULT_FULL_GC_MAX_AGE))); //3. Check that deleted property does get collected post maxAge clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta); @@ -1769,7 +1799,7 @@ public Iterable getModifiedDocs(long fromModified, long toModified assertNotNull(document.get(SETTINGS_COLLECTION_FULL_GC_TIMESTAMP_PROP)); assertNotNull(document.get(SETTINGS_COLLECTION_FULL_GC_DOCUMENT_ID_PROP)); - stats = gcRef.get().gc(maxAge*2, HOURS); + stats = gc(gcRef.get(), maxAge*2, HOURS); document = store1.getDocumentStore().find(SETTINGS, SETTINGS_COLLECTION_ID); assertEquals(5, stats.updatedFullGCDocsCount); @@ -4183,11 +4213,10 @@ private void lateWrite(Collection paths, LateWriteChangesBuilder lateWri * each one in a separate merge * @param orphans the nodes that should be created inproperly - * each one in a separate late-write way - * @param expectedNumOrphanedDocs the expected number of orphan documents that - * FullGC should cleanup * @param unrelatedPath an unrelated path that should be merged after * late-write - ensures lastRev is updated on * root to allow detecting late-writes as such + * @param counts the expected counts of deleted documents array */ private void doLateWriteCreateChildrenGC(Collection parents, Collection orphans, String unrelatedPath, GCCounts... counts) diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java index f7577454b1a..1aa9d47c6dd 100755 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilderTest.java @@ -114,4 +114,11 @@ public void fullGCModeHasDefaultValue() { builder.setFullGCMode(3); assertEquals(0, builder.getFullGCMode()); } + + @Test + public void fullGcMaxAgeInSecsHasDefaultValue() { + RDBDocumentNodeStoreBuilder builder = new RDBDocumentNodeStoreBuilder(); + builder.setFullGcMaxAgeMillis(30 * 24 * 60 * 60 * 1000L); + assertEquals(0, builder.getFullGcMaxAgeMillis()); + } }