Skip to content

Commit e1e8374

Browse files
stroomdev66stroomdev66
committed
#4220 Fix index shard creation
1 parent 87036db commit e1e8374

File tree

4 files changed

+133
-13
lines changed

4 files changed

+133
-13
lines changed

‎stroom-app/src/test/java/stroom/index/TestIndexShardWriterImpl.java

+84
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import stroom.test.AbstractCoreIntegrationTest;
4747
import stroom.test.CommonTestControl;
4848
import stroom.test.CommonTestScenarioCreator;
49+
import stroom.util.io.FileUtil;
4950
import stroom.util.shared.ResultPage;
5051
import stroom.util.shared.Selection;
5152

@@ -57,7 +58,9 @@
5758
import java.io.IOException;
5859
import java.nio.file.Files;
5960
import java.nio.file.Path;
61+
import java.nio.file.Paths;
6062
import java.nio.file.attribute.PosixFilePermission;
63+
import java.util.List;
6164
import java.util.Set;
6265

6366
import static org.assertj.core.api.Assertions.assertThat;
@@ -373,6 +376,87 @@ void testPerformance() {
373376
}
374377
}
375378

379+
@Test
380+
void testDelete() {
381+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isZero();
382+
383+
final DocRef indexRef1 = commonTestScenarioCreator.createIndex("TEST_2010",
384+
commonTestScenarioCreator.createIndexFields(),
385+
1000000);
386+
final LuceneIndexDoc index1 = indexStore.readDocument(indexRef1);
387+
final IndexShardKey indexShardKey1 = IndexShardKey.createKey(index1);
388+
389+
final IndexDocument document1 = new IndexDocument();
390+
document1.add(new FieldValue(LuceneIndexField
391+
.builder()
392+
.fldName("SourcePort")
393+
.fldType(FieldType.TEXT)
394+
.analyzerType(AnalyzerType.ALPHA_NUMERIC)
395+
.termPositions(false)
396+
.indexed(true)
397+
.stored(false)
398+
.build(), ValString.create("12345")));
399+
400+
final Selection<IndexShardStatus> deleted = Selection.selectNone();
401+
deleted.add(IndexShardStatus.DELETED);
402+
final FindIndexShardCriteria findDeleted = FindIndexShardCriteria.builder()
403+
.indexShardStatusSet(deleted).build();
404+
405+
for (int j = 0; j < 10; j++) {
406+
// Delete every shard.
407+
indexShardManager.performAction(FindIndexShardCriteria.matchAll(), IndexShardAction.DELETE);
408+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isEqualTo(j);
409+
assertThat(indexShardDao.find(findDeleted).size()).isEqualTo(j);
410+
411+
// Now add more data.
412+
for (int i = 0; i < 10; i++) {
413+
indexer.addDocument(indexShardKey1, document1);
414+
}
415+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isEqualTo(j + 1);
416+
assertThat(indexShardDao.find(findDeleted).size()).isEqualTo(j);
417+
}
418+
}
419+
420+
@Test
421+
void testCorruption() {
422+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isZero();
423+
424+
final DocRef indexRef1 = commonTestScenarioCreator.createIndex("TEST_2010",
425+
commonTestScenarioCreator.createIndexFields(),
426+
1000000);
427+
final LuceneIndexDoc index1 = indexStore.readDocument(indexRef1);
428+
final IndexShardKey indexShardKey1 = IndexShardKey.createKey(index1);
429+
430+
final IndexDocument document1 = new IndexDocument();
431+
document1.add(new FieldValue(LuceneIndexField
432+
.builder()
433+
.fldName("SourcePort")
434+
.fldType(FieldType.TEXT)
435+
.analyzerType(AnalyzerType.ALPHA_NUMERIC)
436+
.termPositions(false)
437+
.indexed(true)
438+
.stored(false)
439+
.build(), ValString.create("12345")));
440+
441+
final List<IndexVolume> volumeList = indexVolumeDao.getAll();
442+
for (int j = 0; j < 10; j++) {
443+
// Flush.
444+
indexShardManager.performAction(FindIndexShardCriteria.matchAll(), IndexShardAction.FLUSH);
445+
446+
// Delete every shard file.
447+
for (final IndexVolume indexVolume : volumeList) {
448+
FileUtil.deleteContents(Paths.get(indexVolume.getPath()));
449+
}
450+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isEqualTo(j);
451+
452+
// Now add more data.
453+
for (int i = 0; i < 10; i++) {
454+
indexer.addDocument(indexShardKey1, document1);
455+
}
456+
assertThat(indexShardDao.find(FindIndexShardCriteria.matchAll()).size()).isEqualTo(j + 1);
457+
}
458+
}
459+
376460
private void checkDocCount(final int expected, final IndexShardWriter indexShardWriter) {
377461
assertThat(indexShardWriter.getDocumentCount()).isEqualTo(expected);
378462
}

‎stroom-core-shared/src/main/java/stroom/index/shared/IndexException.java

-2
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818

1919
public class IndexException extends RuntimeException {
2020

21-
private static final long serialVersionUID = -482925256715483280L;
22-
2321
public IndexException(final Throwable t) {
2422
super(t);
2523
}

‎stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/ActiveShardsCacheImpl.java

+25-11
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import jakarta.inject.Provider;
2020
import jakarta.inject.Singleton;
2121

22+
import java.io.UncheckedIOException;
2223
import java.util.ArrayList;
2324
import java.util.List;
2425
import java.util.concurrent.atomic.AtomicInteger;
@@ -185,25 +186,34 @@ private boolean addDocument(final IndexDocument document,
185186
private boolean addDocument(final IndexDocument document,
186187
final IndexShard indexShard,
187188
final boolean throwException) {
188-
final IndexShardWriter indexShardWriter = indexShardWriterCache.getOrOpenWriter(indexShard.getId());
189189
try {
190-
indexShardWriter.addDocument(document);
191-
return true;
190+
final IndexShardWriter indexShardWriter = indexShardWriterCache.getOrOpenWriter(indexShard.getId());
191+
try {
192+
indexShardWriter.addDocument(document);
193+
return true;
192194

193-
} catch (final ShardFullException e) {
194-
removeActiveShard(indexShard);
195-
indexShardWriterCache.close(indexShardWriter);
195+
} catch (final IndexException | UncheckedIOException e) {
196+
LOGGER.trace(e::getMessage, e);
196197

197-
} catch (final IndexException | IllegalArgumentException e) {
198-
LOGGER.trace(e::getMessage, e);
198+
removeActiveShard(indexShard);
199+
indexShardWriterCache.close(indexShardWriter);
199200

200-
} catch (final RuntimeException e) {
201+
} catch (final RuntimeException e) {
202+
if (throwException) {
203+
LOGGER.error(e::getMessage, e);
204+
throw e;
205+
} else {
206+
LOGGER.debug(e::getMessage, e);
207+
}
208+
}
209+
} catch (final IndexException e) {
201210
if (throwException) {
202211
LOGGER.error(e::getMessage, e);
203212
throw e;
204213
} else {
205214
LOGGER.debug(e::getMessage, e);
206215
}
216+
removeActiveShard(indexShard);
207217
}
208218
return false;
209219
}
@@ -230,11 +240,15 @@ private synchronized List<IndexShard> ensureShards() {
230240

231241
private synchronized void addActiveShard(final IndexShardKey indexShardKey) {
232242
final IndexShard indexShard = createNewShard(indexShardKey);
233-
indexShards.add(indexShard);
243+
final List<IndexShard> list = new ArrayList<>(indexShards);
244+
list.add(indexShard);
245+
indexShards = list;
234246
}
235247

236248
private synchronized void removeActiveShard(final IndexShard indexShard) {
237-
indexShards.remove(indexShard);
249+
final List<IndexShard> list = new ArrayList<>(indexShards);
250+
list.remove(indexShard);
251+
indexShards = list;
238252
}
239253

240254
/**

‎unreleased_changes/20240418_200430_860__4220.md

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
* Issue **#4220** : Fix index shard creation.
2+
3+
4+
```sh
5+
# ********************************************************************************
6+
# Issue title: beta.19 is creating lots of Index Shards
7+
# Issue link: https://github.com/gchq/stroom/issues/4220
8+
# ********************************************************************************
9+
10+
# ONLY the top line will be included as a change entry in the CHANGELOG.
11+
# The entry should be in GitHub flavour markdown and should be written on a SINGLE
12+
# line with no hard breaks. You can have multiple change files for a single GitHub issue.
13+
# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than
14+
# 'Fixed nasty bug'.
15+
#
16+
# Examples of acceptable entries are:
17+
#
18+
#
19+
# * Issue **123** : Fix bug with an associated GitHub issue in this repository
20+
#
21+
# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository
22+
#
23+
# * Fix bug with no associated GitHub issue.
24+
```

0 commit comments

Comments
 (0)