Skip to content

Commit ed21bd8

Browse files
VladyslavHnesVladyslavHnes
authored
DS-2295 fixed TTL for references (#1821)
* DS-2295 fixed TTL for references Signed-off-by: VladyslavHnes <vladyslav.hnes@gmail.com> * DS-2295 fixed TTL for references Signed-off-by: VladyslavHnes <vladyslav.hnes@gmail.com> --------- Signed-off-by: VladyslavHnes <vladyslav.hnes@gmail.com>
1 parent a17c719 commit ed21bd8

3 files changed

Lines changed: 60 additions & 5 deletions

File tree

‎xyz-hub-service/src/main/java/com/here/xyz/hub/config/dynamo/DynamoDataReferenceConfigClient.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import com.here.xyz.util.service.aws.dynamo.IndexDefinition;
3838
import io.vertx.core.Future;
3939
import java.util.AbstractMap.SimpleImmutableEntry;
40+
import java.util.HashMap;
4041
import java.util.List;
4142
import java.util.Map;
4243
import java.util.Map.Entry;
@@ -64,6 +65,8 @@ public final class DynamoDataReferenceConfigClient extends DataReferenceConfigCl
6465

6566
private static final String ID_INDEX_ATTRIBUTE_NAME = "id";
6667

68+
private static final long KEEP_UNTIL_GRACE_PERIOD_SECONDS = 48 * 3600;
69+
6770
private static final IndexDefinition idIndex = new IndexDefinition(ID_INDEX_ATTRIBUTE_NAME);
6871

6972
private final DynamoClient dynamoClient;
@@ -96,10 +99,22 @@ protected Future<UUID> doStore(DataReference dataReference) {
9699
private static Map<String, Object> dynamoItemAsMap(DataReference dataReference) {
97100
Map<String, Object> dataReferenceAsMap = dataReference.toMap();
98101
dataReferenceAsMap.put(SORT_KEY_NAME, sortKeyValue(dataReference));
99-
102+
// Converts ms to seconds (required by DynamoDB TTL) and adds grace period
103+
if (dataReferenceAsMap.get("keepUntil") != null) {
104+
dataReferenceAsMap.put("keepUntil", ((Number) dataReferenceAsMap.get("keepUntil")).longValue() / 1000 + KEEP_UNTIL_GRACE_PERIOD_SECONDS);
105+
}
100106
return dataReferenceAsMap;
101107
}
102108

109+
private static <T> T fromDynamoMap(Map<String, Object> itemAsMap, Class<T> resultItemClass) {
110+
Map<String, Object> mutableMap = new HashMap<>(itemAsMap);
111+
// Subtracts grace period and converts seconds back to ms for the API
112+
if (mutableMap.get("keepUntil") != null) {
113+
mutableMap.put("keepUntil", (((Number) mutableMap.get("keepUntil")).longValue() - KEEP_UNTIL_GRACE_PERIOD_SECONDS) * 1000);
114+
}
115+
return fromMap(mutableMap, resultItemClass);
116+
}
117+
103118
private static String sortKeyValue(DataReference dataReference) {
104119
return SORT_KEY_PATTERN.formatted(dataReference.getEndVersion(), dataReference.getId());
105120
}
@@ -110,7 +125,7 @@ protected Future<Optional<DataReference>> doLoad(UUID id) {
110125
queryIndex(dataReferenceTable, idIndex, id.toString())
111126
.stream()
112127
.findFirst()
113-
.map(item -> fromMap(item.asMap(), DataReference.class))
128+
.map(item -> fromDynamoMap(item.asMap(), DataReference.class))
114129
);
115130
}
116131

@@ -175,7 +190,7 @@ private static QueryFilter toQueryFilter(Entry<String, Object> filteringParamete
175190
private static <T> List<T> itemCollectionToList(ItemCollection<QueryOutcome> itemCollection, Class<T> resultItemClass) {
176191
return StreamSupport.stream(itemCollection.spliterator(), false)
177192
.map(Item::asMap)
178-
.map(itemAsMap -> fromMap(itemAsMap, resultItemClass))
193+
.map(itemAsMap -> fromDynamoMap(itemAsMap, resultItemClass))
179194
.toList();
180195
}
181196

‎xyz-hub-service/src/main/java/com/here/xyz/hub/util/DataReferenceResolver.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import com.here.xyz.hub.config.SpaceConfigClient;
2525
import com.here.xyz.hub.connectors.models.Space;
2626
import com.here.xyz.models.hub.DataReference;
27+
import com.here.xyz.util.service.Core;
2728
import io.vertx.core.Future;
2829
import org.apache.logging.log4j.Marker;
2930

@@ -65,6 +66,10 @@ public Future<Optional<DataReference>> loadById(Marker marker, UUID referenceId,
6566

6667
DataReference ref = maybeRef.get();
6768

69+
if (isExpired(ref)) {
70+
return Future.succeededFuture(Optional.empty());
71+
}
72+
6873
if (onlyStale) {
6974
return Future.succeededFuture(Optional.of(ref));
7075
}
@@ -78,18 +83,19 @@ public Future<List<DataReference>> filterStaleForEntity(Marker marker, String en
7883
}
7984

8085
public Future<List<DataReference>> filterForEntity(Marker marker, String entityId, List<DataReference> refs, boolean onlyStale) {
86+
List<DataReference> nonExpiredRefs = refs.stream().filter(r -> !isExpired(r)).toList();
8187
return resolveAnchorSpace(marker, entityId)
8288
.map(maybeAnchor -> {
8389
if (maybeAnchor.isEmpty()) {
8490
if (onlyStale) {
8591
return List.of();
8692
}
8793

88-
return distinctNewestByUniquenessKey(refs);
94+
return distinctNewestByUniquenessKey(nonExpiredRefs);
8995
}
9096

9197
long minCreatedAt = maybeAnchor.get().createdAt();
92-
List<DataReference> filtered = refs.stream()
98+
List<DataReference> filtered = nonExpiredRefs.stream()
9399
.filter(r -> onlyStale
94100
? ts(r.getCreatedAt()) < minCreatedAt
95101
: ts(r.getCreatedAt()) >= minCreatedAt)
@@ -224,6 +230,10 @@ private static long ts(Long v) {
224230
return v == null ? 0L : v;
225231
}
226232

233+
private static boolean isExpired(DataReference r) {
234+
return r.getKeepUntil() != null && r.getKeepUntil() < Core.currentTimeMillis();
235+
}
236+
227237
private record Anchor(String spaceId, long createdAt) {}
228238

229239
private record UniquenessKey(

‎xyz-hub-service/src/test/java/com/here/xyz/hub/util/DataReferenceResolverTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,36 @@ void filterForEntity_shouldReturnDistinctNewestPerUniquenessKey_whenAnchorIsMiss
370370
assertThat(result).containsExactlyInAnyOrder(keyANew, keyB);
371371
}
372372

373+
@Test
374+
void loadById_shouldReturnEmpty_whenReferenceIsExpired() {
375+
UUID id = UUID.randomUUID();
376+
long expiredKeepUntil = System.currentTimeMillis() - 1000;
377+
DataReference expired = ref("entity-id-1", 100L).withId(id).withKeepUntil(expiredKeepUntil);
378+
379+
when(references.load(id)).thenReturn(Future.succeededFuture(Optional.of(expired)));
380+
381+
Optional<DataReference> result = await(resolver.loadById(marker, id, false));
382+
383+
assertThat(result).isEmpty();
384+
verifyNoInteractions(spaces);
385+
}
386+
387+
@Test
388+
void filterForEntity_shouldExcludeExpiredReferences() {
389+
String entityId = "entity-id-1";
390+
long expiredKeepUntil = System.currentTimeMillis() - 1000;
391+
long futureKeepUntil = System.currentTimeMillis() + 60_000;
392+
393+
DataReference expired = ref(entityId, 100L).withKeepUntil(expiredKeepUntil);
394+
DataReference valid = ref(entityId, 200L).withKeepUntil(futureKeepUntil);
395+
396+
when(spaces.get(marker, entityId)).thenReturn(Future.succeededFuture(null));
397+
398+
List<DataReference> result = await(resolver.filterForEntity(marker, entityId, List.of(expired, valid), false));
399+
400+
assertThat(result).containsExactly(valid);
401+
}
402+
373403
private static DataReference ref(String entityId, Long createdAt) {
374404
DataReference r = new DataReference().withEntityId(entityId);
375405
if (createdAt != null) {

0 commit comments

Comments
 (0)