|
37 | 37 | import com.linkedin.davinci.stats.ingestion.heartbeat.HeartbeatMonitoringService; |
38 | 38 | import com.linkedin.davinci.storage.StorageMetadataService; |
39 | 39 | import com.linkedin.davinci.storage.StorageService; |
| 40 | +import com.linkedin.davinci.storage.chunking.ChunkedValueManifestContainer; |
40 | 41 | import com.linkedin.davinci.store.DelegatingStorageEngine; |
41 | 42 | import com.linkedin.davinci.store.view.MaterializedViewWriter; |
42 | 43 | import com.linkedin.davinci.store.view.VeniceViewWriter; |
|
45 | 46 | import com.linkedin.davinci.validation.PartitionTracker; |
46 | 47 | import com.linkedin.venice.compression.CompressionStrategy; |
47 | 48 | import com.linkedin.venice.compression.VeniceCompressor; |
| 49 | +import com.linkedin.venice.exceptions.VeniceException; |
48 | 50 | import com.linkedin.venice.exceptions.VeniceTimeoutException; |
49 | 51 | import com.linkedin.venice.kafka.protocol.ControlMessage; |
50 | 52 | import com.linkedin.venice.kafka.protocol.Delete; |
|
96 | 98 | import com.linkedin.venice.writer.VeniceWriter; |
97 | 99 | import it.unimi.dsi.fastutil.objects.Object2IntMaps; |
98 | 100 | import java.io.IOException; |
| 101 | +import java.lang.reflect.Field; |
99 | 102 | import java.nio.ByteBuffer; |
100 | 103 | import java.util.Collections; |
101 | 104 | import java.util.HashMap; |
@@ -1397,4 +1400,97 @@ public void testDolStampIsDolComplete() { |
1397 | 1400 | dolStampIncomplete.setDolConsumed(true); |
1398 | 1401 | assertFalse(dolStampIncomplete.isDolComplete()); |
1399 | 1402 | } |
| 1403 | + |
| 1404 | + /** |
| 1405 | + * Tests that {@link StoreIngestionTask#putGlobalRtDivStateInMetadata} throws a VeniceException |
| 1406 | + * when the key does not start with the expected prefix, and delegates to storageMetadataService |
| 1407 | + * when the key is valid. |
| 1408 | + */ |
| 1409 | + @Test |
| 1410 | + public void testPutGlobalRtDivStateInMetadata() throws Exception { |
| 1411 | + LeaderFollowerStoreIngestionTask ingestionTask = mock(LeaderFollowerStoreIngestionTask.class); |
| 1412 | + doCallRealMethod().when(ingestionTask).putGlobalRtDivStateInMetadata(anyInt(), any(), any()); |
| 1413 | + |
| 1414 | + Put put = new Put(); |
| 1415 | + put.putValue = ByteBuffer.wrap("test-value".getBytes()); |
| 1416 | + |
| 1417 | + // Invalid key (missing prefix) should throw VeniceException immediately, before any field access |
| 1418 | + byte[] invalidKey = "INVALID_KEY.localhost:9092".getBytes(); |
| 1419 | + Assert.assertThrows(VeniceException.class, () -> ingestionTask.putGlobalRtDivStateInMetadata(0, invalidKey, put)); |
| 1420 | + |
| 1421 | + // Valid key should call storageMetadataService.putGlobalRtDivState |
| 1422 | + StorageMetadataService mockSms = mock(StorageMetadataService.class); |
| 1423 | + String versionTopic = "testStore_v1"; |
| 1424 | + injectField(ingestionTask, StoreIngestionTask.class, "storageMetadataService", mockSms); |
| 1425 | + injectField(ingestionTask, StoreIngestionTask.class, "kafkaVersionTopic", versionTopic); |
| 1426 | + |
| 1427 | + String brokerUrl = "localhost:9092"; |
| 1428 | + byte[] validKey = (StoreIngestionTask.GLOBAL_RT_DIV_KEY_PREFIX + brokerUrl).getBytes(); |
| 1429 | + ingestionTask.putGlobalRtDivStateInMetadata(0, validKey, put); |
| 1430 | + |
| 1431 | + ArgumentCaptor<byte[]> valueCaptor = ArgumentCaptor.forClass(byte[].class); |
| 1432 | + verify(mockSms, times(1)).putGlobalRtDivState(eq(versionTopic), eq(0), eq(brokerUrl), valueCaptor.capture()); |
| 1433 | + } |
| 1434 | + |
| 1435 | + /** |
| 1436 | + * Tests the new metadata-first lookup in {@link LeaderFollowerStoreIngestionTask#readGlobalRtDivState}: |
| 1437 | + * - When metadata storage has a value, it is returned without hitting legacy storage. |
| 1438 | + * - When metadata storage is empty, the method falls through to the legacy path. |
| 1439 | + * - When the key does not start with the GLOBAL_RT_DIV_KEY_PREFIX, the metadata lookup is skipped. |
| 1440 | + */ |
| 1441 | + @Test |
| 1442 | + public void testReadGlobalRtDivStateMetadataPath() throws Exception { |
| 1443 | + LeaderFollowerStoreIngestionTask ingestionTask = mock(LeaderFollowerStoreIngestionTask.class); |
| 1444 | + doCallRealMethod().when(ingestionTask) |
| 1445 | + .readGlobalRtDivState(any(), anyInt(), any(), any(ChunkedValueManifestContainer.class)); |
| 1446 | + |
| 1447 | + StorageMetadataService mockSms = mock(StorageMetadataService.class); |
| 1448 | + String versionTopic = "testStore_v1"; |
| 1449 | + InternalAvroSpecificSerializer<GlobalRtDivState> serializer = |
| 1450 | + AvroProtocolDefinition.GLOBAL_RT_DIV_STATE.getSerializer(); |
| 1451 | + |
| 1452 | + injectField(ingestionTask, StoreIngestionTask.class, "storageMetadataService", mockSms); |
| 1453 | + injectField(ingestionTask, StoreIngestionTask.class, "kafkaVersionTopic", versionTopic); |
| 1454 | + injectField(ingestionTask, LeaderFollowerStoreIngestionTask.class, "globalRtDivStateSerializer", serializer); |
| 1455 | + |
| 1456 | + String brokerUrl = "localhost:9092"; |
| 1457 | + byte[] keyBytes = (StoreIngestionTask.GLOBAL_RT_DIV_KEY_PREFIX + brokerUrl).getBytes(); |
| 1458 | + PubSubTopicPartition topicPartition = mock(PubSubTopicPartition.class); |
| 1459 | + doReturn(0).when(topicPartition).getPartitionNumber(); |
| 1460 | + ChunkedValueManifestContainer manifestContainer = new ChunkedValueManifestContainer(); |
| 1461 | + |
| 1462 | + // Serialize a real GlobalRtDivState for the round-trip test |
| 1463 | + GlobalRtDivState expectedState = |
| 1464 | + new GlobalRtDivState(brokerUrl, Collections.emptyMap(), InMemoryPubSubPosition.of(5).toWireFormatBuffer()); |
| 1465 | + byte[] serializedState = serializer.serialize(null, expectedState); |
| 1466 | + |
| 1467 | + // Case 1: metadata present → returns deserialized state without touching legacy storage |
| 1468 | + doReturn(Optional.of(serializedState)).when(mockSms).getGlobalRtDivState(versionTopic, 0, brokerUrl); |
| 1469 | + GlobalRtDivState result = |
| 1470 | + ingestionTask.readGlobalRtDivState(keyBytes, GLOBAL_RT_DIV_VERSION, topicPartition, manifestContainer); |
| 1471 | + assertNotNull(result); |
| 1472 | + assertEquals(result.srcUrl.toString(), brokerUrl); |
| 1473 | + |
| 1474 | + // Case 2: metadata absent → falls through to legacy; legacy returns null (compressor field is null on mock), |
| 1475 | + // so the method returns null |
| 1476 | + doReturn(Optional.empty()).when(mockSms).getGlobalRtDivState(versionTopic, 0, brokerUrl); |
| 1477 | + GlobalRtDivState fallThroughResult = |
| 1478 | + ingestionTask.readGlobalRtDivState(keyBytes, GLOBAL_RT_DIV_VERSION, topicPartition, manifestContainer); |
| 1479 | + Assert.assertNull(fallThroughResult); |
| 1480 | + |
| 1481 | + // Case 3: key does not start with the prefix → metadata lookup skipped, goes straight to legacy (returns null) |
| 1482 | + byte[] nonPrefixKey = "REGULAR_KEY.localhost:9092".getBytes(); |
| 1483 | + GlobalRtDivState nonPrefixResult = |
| 1484 | + ingestionTask.readGlobalRtDivState(nonPrefixKey, GLOBAL_RT_DIV_VERSION, topicPartition, manifestContainer); |
| 1485 | + Assert.assertNull(nonPrefixResult); |
| 1486 | + // storageMetadataService.getGlobalRtDivState should NOT have been called for the non-prefix key |
| 1487 | + verify(mockSms, times(0)).getGlobalRtDivState(versionTopic, 0, "REGULAR_KEY.localhost:9092".substring(0)); |
| 1488 | + } |
| 1489 | + |
| 1490 | + private static void injectField(Object target, Class<?> declaringClass, String fieldName, Object value) |
| 1491 | + throws Exception { |
| 1492 | + Field field = declaringClass.getDeclaredField(fieldName); |
| 1493 | + field.setAccessible(true); |
| 1494 | + field.set(target, value); |
| 1495 | + } |
1400 | 1496 | } |
0 commit comments