Skip to content

Commit 0d391f6

Browse files
smiroslavsmiroslav
authored
OAK-11569 - Revert OAK-11287: Cleanup May Delete Referenced Segments (#1880) (#2156)
This reverts commit bc67ca7.
1 parent f611f8b commit 0d391f6

17 files changed

+499
-452
lines changed

‎oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import com.microsoft.azure.storage.blob.CloudBlobDirectory;
3434
import com.microsoft.azure.storage.blob.ListBlobItem;
3535

36-
import org.apache.jackrabbit.oak.segment.RecordId;
3736
import org.apache.jackrabbit.oak.segment.SegmentCache;
3837
import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
3938
import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
@@ -369,8 +368,8 @@ public int run() throws IOException, StorageException, URISyntaxException {
369368
targetContainer = destinationCloudBlobDirectory.getContainer();
370369
}
371370

372-
GCGeneration gcGeneration;
373-
RecordId root;
371+
GCGeneration gcGeneration = null;
372+
String root = null;
374373

375374
try (FileStore store =newFileStore(splitPersistence,
376375
Files.createTempDirectory(getClass().getSimpleName() + "-").toFile(),
@@ -403,7 +402,7 @@ public int run() throws IOException, StorageException, URISyntaxException {
403402

404403
System.out.printf(" -> [skipping] cleaning up\n");
405404
gcGeneration = store.getHead().getGcGeneration();
406-
root = store.getHead().getRecordId();
405+
root = store.getHead().getRecordId().toString10();
407406
} catch (Exception e) {
408407
watch.stop();
409408
e.printStackTrace(System.err);
@@ -430,7 +429,7 @@ public int run() throws IOException, StorageException, URISyntaxException {
430429
return 0;
431430
}
432431

433-
private void persistGCJournal(SegmentNodeStorePersistence rwPersistence, long newSize, GCGeneration gcGeneration, RecordId root) throws IOException {
432+
private void persistGCJournal(SegmentNodeStorePersistence rwPersistence, long newSize, GCGeneration gcGeneration, String root) throws IOException {
434433
GCJournalFile gcJournalFile = rwPersistence.getGCJournalFile();
435434
if (gcJournalFile != null) {
436435
GCJournal gcJournal = new GCJournal(gcJournalFile);

‎oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java‎

Lines changed: 327 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,327 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.jackrabbit.oak.segment.file;
20+
21+
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.ESTIMATION;
22+
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.IDLE;
23+
24+
import java.io.IOException;
25+
import java.util.List;
26+
import java.util.function.Predicate;
27+
import java.util.function.Supplier;
28+
29+
import org.apache.jackrabbit.oak.segment.Revisions;
30+
import org.apache.jackrabbit.oak.segment.SegmentCache;
31+
import org.apache.jackrabbit.oak.segment.SegmentReader;
32+
import org.apache.jackrabbit.oak.segment.SegmentTracker;
33+
import org.apache.jackrabbit.oak.segment.SegmentWriterFactory;
34+
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
35+
import org.apache.jackrabbit.oak.segment.file.cancel.Canceller;
36+
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
37+
import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
38+
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
39+
40+
abstract class AbstractGarbageCollectionStrategy implements GarbageCollectionStrategy {
41+
42+
abstract EstimationStrategy getFullEstimationStrategy();
43+
44+
abstract EstimationStrategy getTailEstimationStrategy();
45+
46+
abstract CompactionStrategy getFullCompactionStrategy();
47+
48+
abstract CompactionStrategy getTailCompactionStrategy();
49+
50+
abstract CleanupStrategy getCleanupStrategy();
51+
52+
@Override
53+
public void collectGarbage(Context context) throws IOException {
54+
switch (context.getGCOptions().getGCType()) {
55+
case FULL:
56+
collectFullGarbage(context);
57+
break;
58+
case TAIL:
59+
collectTailGarbage(context);
60+
break;
61+
default:
62+
throw new IllegalStateException("Invalid GC type");
63+
}
64+
}
65+
66+
@Override
67+
public void collectFullGarbage(Context context) throws IOException {
68+
run(context, getFullEstimationStrategy(), getFullCompactionStrategy());
69+
}
70+
71+
@Override
72+
public void collectTailGarbage(Context context) throws IOException {
73+
run(context, getTailEstimationStrategy(), getTailCompactionStrategy());
74+
}
75+
76+
@Override
77+
public CompactionResult compactFull(Context context) throws IOException {
78+
return getFullCompactionStrategy().compact(newCompactionStrategyContext(context));
79+
}
80+
81+
@Override
82+
public CompactionResult compactTail(Context context) throws IOException {
83+
return getTailCompactionStrategy().compact(newCompactionStrategyContext(context));
84+
}
85+
86+
@Override
87+
public List<String> cleanup(Context context) throws IOException {
88+
return cleanup(context, CompactionResult.skipped(
89+
context.getLastCompactionType(),
90+
getGcGeneration(context),
91+
context.getGCOptions(),
92+
context.getRevisions().getHead(),
93+
context.getGCCount()
94+
));
95+
}
96+
97+
void run(Context context, EstimationStrategy estimationStrategy, CompactionStrategy compactionStrategy) throws IOException {
98+
try {
99+
context.getGCListener().info("started");
100+
101+
long dt = System.currentTimeMillis() - context.getLastSuccessfulGC();
102+
103+
if (dt < context.getGCBackOff()) {
104+
context.getGCListener().skipped("skipping garbage collection as it already ran less than {} hours ago ({} s).", context.getGCBackOff() / 3600000, dt / 1000);
105+
return;
106+
}
107+
108+
boolean sufficientEstimatedGain = true;
109+
if (context.getGCOptions().isEstimationDisabled()) {
110+
context.getGCListener().info("estimation skipped because it was explicitly disabled");
111+
} else if (context.getGCOptions().isPaused()) {
112+
context.getGCListener().info("estimation skipped because compaction is paused");
113+
} else {
114+
context.getGCListener().info("estimation started");
115+
context.getGCListener().updateStatus(ESTIMATION.message());
116+
117+
PrintableStopwatch watch = PrintableStopwatch.createStarted();
118+
EstimationResult estimation = estimationStrategy.estimate(newEstimationStrategyContext(context));
119+
sufficientEstimatedGain = estimation.isGcNeeded();
120+
String gcLog = estimation.getGcLog();
121+
if (sufficientEstimatedGain) {
122+
context.getGCListener().info("estimation completed in {}. {}", watch, gcLog);
123+
} else {
124+
context.getGCListener().skipped("estimation completed in {}. {}", watch, gcLog);
125+
}
126+
}
127+
128+
if (sufficientEstimatedGain) {
129+
try (GCMemoryBarrier ignored = new GCMemoryBarrier(context.getSufficientMemory(), context.getGCListener(), context.getGCOptions())) {
130+
if (context.getGCOptions().isPaused()) {
131+
context.getGCListener().skipped("compaction paused");
132+
} else if (!context.getSufficientMemory().get()) {
133+
context.getGCListener().skipped("compaction skipped. Not enough memory");
134+
} else {
135+
CompactionResult compactionResult = compactionStrategy.compact(newCompactionStrategyContext(context));
136+
if (compactionResult.isSuccess()) {
137+
context.getSuccessfulGarbageCollectionListener().onSuccessfulGarbageCollection();
138+
} else {
139+
context.getGCListener().info("cleaning up after failed compaction");
140+
}
141+
context.getFileReaper().add(cleanup(context, compactionResult));
142+
}
143+
}
144+
}
145+
} finally {
146+
context.getCompactionMonitor().finished();
147+
context.getGCListener().updateStatus(IDLE.message());
148+
}
149+
}
150+
151+
private GCGeneration getGcGeneration(Context context) {
152+
return context.getRevisions().getHead().getSegmentId().getGcGeneration();
153+
}
154+
155+
public List<String> cleanup(Context context, CompactionResult compactionResult) throws IOException {
156+
return getCleanupStrategy().cleanup(newCleanupStrategyContext(context, compactionResult));
157+
}
158+
159+
private EstimationStrategy.Context newEstimationStrategyContext(Context context) {
160+
return new EstimationStrategy.Context() {
161+
162+
@Override
163+
public long getSizeDelta() {
164+
return context.getGCOptions().getGcSizeDeltaEstimation();
165+
}
166+
167+
@Override
168+
public long getCurrentSize() {
169+
return context.getTarFiles().size();
170+
}
171+
172+
@Override
173+
public GCJournal getGCJournal() {
174+
return context.getGCJournal();
175+
}
176+
177+
};
178+
}
179+
180+
private static CompactionStrategy.Context newCompactionStrategyContext(Context context) {
181+
return new CompactionStrategy.Context() {
182+
183+
@Override
184+
public SegmentTracker getSegmentTracker() {
185+
return context.getSegmentTracker();
186+
}
187+
188+
@Override
189+
public GCListener getGCListener() {
190+
return context.getGCListener();
191+
}
192+
193+
@Override
194+
public GCJournal getGCJournal() {
195+
return context.getGCJournal();
196+
}
197+
198+
@Override
199+
public SegmentGCOptions getGCOptions() {
200+
return context.getGCOptions();
201+
}
202+
203+
@Override
204+
public GCNodeWriteMonitor getCompactionMonitor() {
205+
return context.getCompactionMonitor();
206+
}
207+
208+
@Override
209+
public SegmentReader getSegmentReader() {
210+
return context.getSegmentReader();
211+
}
212+
213+
@Override
214+
public SegmentWriterFactory getSegmentWriterFactory() {
215+
return context.getSegmentWriterFactory();
216+
}
217+
218+
@Override
219+
public Revisions getRevisions() {
220+
return context.getRevisions();
221+
}
222+
223+
@Override
224+
public TarFiles getTarFiles() {
225+
return context.getTarFiles();
226+
}
227+
228+
@Override
229+
public BlobStore getBlobStore() {
230+
return context.getBlobStore();
231+
}
232+
233+
@Override
234+
public Canceller getHardCanceller() {
235+
return context.getCanceller();
236+
}
237+
238+
@Override
239+
public Canceller getSoftCanceller() {
240+
return Canceller.newCanceller();
241+
}
242+
243+
@Override
244+
public Supplier<Canceller> getStateSaveTriggerSupplier() {
245+
return Canceller::newCanceller;
246+
}
247+
248+
@Override
249+
public int getGCCount() {
250+
return context.getGCCount();
251+
}
252+
253+
@Override
254+
public SuccessfulCompactionListener getSuccessfulCompactionListener() {
255+
return context.getSuccessfulCompactionListener();
256+
}
257+
258+
@Override
259+
public Flusher getFlusher() {
260+
return context.getFlusher();
261+
}
262+
263+
};
264+
}
265+
266+
private CleanupStrategy.Context newCleanupStrategyContext(Context context, CompactionResult compactionResult) {
267+
return new CleanupStrategy.Context() {
268+
269+
@Override
270+
public GCListener getGCListener() {
271+
return context.getGCListener();
272+
}
273+
274+
@Override
275+
public SegmentCache getSegmentCache() {
276+
return context.getSegmentCache();
277+
}
278+
279+
@Override
280+
public SegmentTracker getSegmentTracker() {
281+
return context.getSegmentTracker();
282+
}
283+
284+
@Override
285+
public FileStoreStats getFileStoreStats() {
286+
return context.getFileStoreStats();
287+
}
288+
289+
@Override
290+
public GCNodeWriteMonitor getCompactionMonitor() {
291+
return context.getCompactionMonitor();
292+
}
293+
294+
@Override
295+
public GCJournal getGCJournal() {
296+
return compactionResult.requiresGCJournalEntry() ? context.getGCJournal() : null;
297+
}
298+
299+
@Override
300+
public Predicate<GCGeneration> getReclaimer() {
301+
return compactionResult.reclaimer();
302+
}
303+
304+
@Override
305+
public TarFiles getTarFiles() {
306+
return context.getTarFiles();
307+
}
308+
309+
@Override
310+
public Revisions getRevisions() {
311+
return context.getRevisions();
312+
}
313+
314+
@Override
315+
public String getCompactedRootId() {
316+
return compactionResult.getCompactedRootId().toString10();
317+
}
318+
319+
@Override
320+
public String getSegmentEvictionReason() {
321+
return compactionResult.gcInfo();
322+
}
323+
324+
};
325+
}
326+
327+
}

0 commit comments

Comments
 (0)