Skip to content

Commit 2dc7e58

Browse files
j-collj-coll
authored
Merge pull request #2637 from opencb/TASK-6771
TASK-6771 - Error running `variant-file-delete` : IllegalArgumentException: Unknown status ERROR
2 parents 48b6126 + 043e911 commit 2dc7e58

3 files changed

Lines changed: 96 additions & 12 deletions

File tree

‎opencga-analysis/src/main/java/org/opencb/opencga/analysis/variant/metadata/CatalogStorageMetadataSynchronizer.java‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,7 +1063,7 @@ private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMeta
10631063
catalogManager.getSampleManager()
10641064
.updateSampleInternalVariantIndex(study.getName(), sample,
10651065
new SampleInternalVariantIndex(
1066-
new IndexStatus(sampleMetadata.getIndexStatus().name()),
1066+
toIndexStatus(sampleMetadata.getIndexStatus()),
10671067
sampleMetadata.getFiles().size(),
10681068
sampleMetadata.isMultiFileSample()), token);
10691069
modified = true;
@@ -1073,7 +1073,7 @@ private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMeta
10731073
catalogManager.getSampleManager()
10741074
.updateSampleInternalVariantAnnotationIndex(study.getName(), sample,
10751075
new SampleInternalVariantAnnotationIndex(
1076-
new IndexStatus(sampleMetadata.getAnnotationStatus().name())), token);
1076+
toIndexStatus(sampleMetadata.getAnnotationStatus())), token);
10771077
modified = true;
10781078
}
10791079

@@ -1083,7 +1083,7 @@ private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMeta
10831083
catalogManager.getSampleManager()
10841084
.updateSampleInternalVariantSecondaryAnnotationIndex(study.getName(), sample,
10851085
new SampleInternalVariantSecondaryAnnotationIndex(
1086-
new IndexStatus(sampleMetadata.getSecondaryAnnotationIndexStatus().name())), token);
1086+
toIndexStatus(sampleMetadata.getSecondaryAnnotationIndexStatus())), token);
10871087
modified = true;
10881088
}
10891089

@@ -1136,14 +1136,15 @@ private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMeta
11361136
catalogVariantSecondarySampleIndexModified = true;
11371137
}
11381138

1139-
String sampleIndexFamilyStatus = sampleMetadata.getFamilyIndexStatus(sampleIndexVersion).name();
1140-
String catalogSecondarySampleIndexFamilyStatus = secureGet(sample, s -> s.getInternal().getVariant().getSecondarySampleIndex().getFamilyStatus().getId(), null);
1141-
if (!sampleIndexFamilyStatus.equals(catalogSecondarySampleIndexFamilyStatus)) {
1139+
IndexStatus sampleIndexFamilyStatus = toIndexStatus(sampleMetadata.getFamilyIndexStatus(sampleIndexVersion));
1140+
String catalogSecondarySampleIndexFamilyStatus = secureGet(sample, s -> s.getInternal().getVariant()
1141+
.getSecondarySampleIndex().getFamilyStatus().getId(), null);
1142+
if (!sampleIndexFamilyStatus.getId().equals(catalogSecondarySampleIndexFamilyStatus)) {
11421143
String message = "Family Index is "
1143-
+ (sampleIndexFamilyStatus.equals(IndexStatus.READY) ? "ready" : "not ready")
1144+
+ (sampleIndexFamilyStatus.getId().equals(IndexStatus.READY) ? "ready" : "not ready")
11441145
+ " with version=" + sampleIndexVersion;
11451146
catalogVariantSecondarySampleIndex.setFamilyStatus(
1146-
new IndexStatus(sampleIndexFamilyStatus, message));
1147+
new IndexStatus(sampleIndexFamilyStatus.getId(), message));
11471148
catalogVariantSecondarySampleIndexModified = true;
11481149
}
11491150

@@ -1214,6 +1215,41 @@ private boolean synchronizeSample(StudyMetadata study, SampleMetadata sampleMeta
12141215
return modified;
12151216
}
12161217

1218+
private static IndexStatus toIndexStatus(TaskMetadata.Status indexStatus) {
1219+
return toIndexStatus(indexStatus, "");
1220+
}
1221+
1222+
private static IndexStatus toIndexStatus(TaskMetadata.Status storageStatus, String message) {
1223+
String statusName;
1224+
switch (storageStatus) {
1225+
case NONE:
1226+
case ERROR:
1227+
case ABORTED:
1228+
statusName = IndexStatus.NONE;
1229+
break;
1230+
case RUNNING:
1231+
statusName = IndexStatus.INDEXING;
1232+
break;
1233+
case DONE:
1234+
case READY:
1235+
statusName = IndexStatus.READY;
1236+
break;
1237+
case INVALID:
1238+
statusName = IndexStatus.INVALID;
1239+
break;
1240+
default:
1241+
throw new IllegalStateException("Unexpected value: " + storageStatus);
1242+
}
1243+
if (message.isEmpty()) {
1244+
if (statusName.equals(storageStatus.name())) {
1245+
message = "Index is " + statusName.toLowerCase();
1246+
} else {
1247+
message = "Index is " + statusName.toLowerCase() + " (storage status: " + storageStatus.name().toLowerCase() + ")";
1248+
}
1249+
}
1250+
return new IndexStatus(statusName, message);
1251+
}
1252+
12171253
public void synchronizeRemovedStudyFromStorage(String study, String token) throws CatalogException {
12181254
catalogManager.getCohortManager().update(study, StudyEntry.DEFAULT_COHORT,
12191255
new CohortUpdateParams().setSamples(Collections.emptyList()),

‎opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageEngine.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,22 +1184,41 @@ private void remove(String study, List<String> files, List<String> samples, URI
11841184
service.awaitTermination(10, TimeUnit.DAYS);
11851185
if (!samplesToRebuildIndex.isEmpty()) {
11861186
logger.info("Rebuild sample index for samples " + samplesToRebuildIndex);
1187+
List<Trio> samplesToRebuildFamilyIndex = new ArrayList<>();
11871188
for (String sample : samplesToRebuildIndex) {
11881189
int sampleId = getMetadataManager().getSampleIdOrFail(studyId, sample);
11891190
getMetadataManager().updateSampleMetadata(studyId, sampleId, sampleMetadata -> {
11901191
for (int v : sampleMetadata.getSampleIndexVersions()) {
11911192
sampleMetadata.setSampleIndexStatus(TaskMetadata.Status.ERROR, v);
11921193
}
11931194
for (int v : sampleMetadata.getSampleIndexAnnotationVersions()) {
1194-
sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.ERROR, v);
1195+
if (sampleMetadata.getSampleIndexAnnotationStatus(v) == TaskMetadata.Status.READY) {
1196+
sampleMetadata.setSampleIndexAnnotationStatus(TaskMetadata.Status.ERROR, v);
1197+
}
11951198
}
1199+
boolean hasReadyFamilyIndex = false;
11961200
for (int v : sampleMetadata.getFamilyIndexVersions()) {
1197-
sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.ERROR, v);
1201+
if (sampleMetadata.getFamilyIndexStatus(v) == TaskMetadata.Status.READY) {
1202+
hasReadyFamilyIndex = true;
1203+
sampleMetadata.setFamilyIndexStatus(TaskMetadata.Status.ERROR, v);
1204+
}
1205+
}
1206+
if (hasReadyFamilyIndex) {
1207+
String fatherName = sampleMetadata.getFather() != null
1208+
? getMetadataManager().getSampleName(studyId, sampleMetadata.getFather())
1209+
: null;
1210+
String motherName = sampleMetadata.getMother() != null
1211+
? getMetadataManager().getSampleName(studyId, sampleMetadata.getMother())
1212+
: null;
1213+
samplesToRebuildFamilyIndex.add(new Trio(fatherName, motherName, sampleMetadata.getName()));
11981214
}
11991215
});
12001216
}
12011217
sampleIndex(study, samplesToRebuildIndex, options);
12021218
sampleIndexAnnotate(study, samplesToRebuildIndex, options);
1219+
if (!samplesToRebuildFamilyIndex.isEmpty()) {
1220+
familyIndex(study, samplesToRebuildFamilyIndex, options);
1221+
}
12031222
}
12041223

12051224
logger.info("------------------------------------------------------");

‎opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/VariantTableRemoveTest.java‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,7 @@
3131
import org.opencb.opencga.core.api.ParamConstants;
3232
import org.opencb.opencga.core.models.operations.variant.VariantAggregateFamilyParams;
3333
import org.opencb.opencga.core.testclassification.duration.MediumTests;
34-
import org.opencb.opencga.storage.core.metadata.models.FileMetadata;
35-
import org.opencb.opencga.storage.core.metadata.models.StudyMetadata;
34+
import org.opencb.opencga.storage.core.metadata.models.*;
3635
import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest;
3736
import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
3837
import org.opencb.opencga.storage.core.variant.VariantStorageOptions;
@@ -166,13 +165,43 @@ public void removeFileTestMergeBasic() throws Exception {
166165

167166
@Test
168167
public void removeFileTestMultiFile() throws Exception {
168+
removeFileTestMultiFile(false, false);
169+
}
170+
171+
@Test
172+
public void removeFileTestMultiFileWithAnnotationAndFamilyIndex() throws Exception {
173+
removeFileTestMultiFile(true, true);
174+
}
175+
176+
public void removeFileTestMultiFile(boolean annotate, boolean familyIndex) throws Exception {
169177
StudyMetadata studyMetadata = VariantStorageBaseTest.newStudyMetadata();
170178

171179
loadFile("s1.genome.vcf", studyMetadata, new ObjectMap());
172180
loadFile("s2.genome.vcf", studyMetadata, new ObjectMap());
173181
loadFile("s1_s2.genome.vcf", studyMetadata, new ObjectMap(VariantStorageOptions.LOAD_MULTI_FILE_DATA.key(), true));
182+
if (annotate) {
183+
variantStorageEngine.annotate(newOutputUri(), new ObjectMap());
184+
}
185+
if (familyIndex) {
186+
variantStorageEngine.familyIndex(studyMetadata.getName(), Collections.singletonList(new Trio("s1", null, "s2")), new ObjectMap());
187+
}
188+
int sampleId = metadataManager.getSampleIdOrFail(studyMetadata.getId(), "s2");
189+
190+
SampleMetadata sampleMetadata = metadataManager.getSampleMetadata(studyMetadata.getId(), sampleId);
191+
// Ensure sample index status is ready
192+
assertEquals(TaskMetadata.Status.READY, sampleMetadata.getSampleIndexStatus(1));
193+
assertEquals(annotate ? TaskMetadata.Status.READY : TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1));
194+
assertEquals(familyIndex ? TaskMetadata.Status.READY : TaskMetadata.Status.NONE, sampleMetadata.getFamilyIndexStatus(1));
195+
174196
VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri());
175197
removeFile("s2.genome.vcf", studyMetadata, Collections.emptyMap());
198+
199+
sampleMetadata = metadataManager.getSampleMetadata(studyMetadata.getId(), sampleId);
200+
// Ensure sample index status is still ready
201+
assertEquals(TaskMetadata.Status.READY, sampleMetadata.getSampleIndexStatus(1));
202+
assertEquals(annotate ? TaskMetadata.Status.READY : TaskMetadata.Status.NONE, sampleMetadata.getSampleIndexAnnotationStatus(1));
203+
assertEquals(familyIndex ? TaskMetadata.Status.READY : TaskMetadata.Status.NONE, sampleMetadata.getFamilyIndexStatus(1));
204+
176205
VariantHbaseTestUtils.printVariants(getVariantStorageEngine().getDBAdaptor(), newOutputUri());
177206
variantStorageEngine.calculateStats(studyMetadata.getName(), Collections.singletonList(StudyEntry.DEFAULT_COHORT), new QueryOptions());
178207

0 commit comments

Comments
 (0)