Skip to content

Commit 7421dcb

Browse files
sumanth-pasupuletisumanth-pasupuleti
authored
Merge pull request #89 from Netflix/feature/consolidate_hashing
Consolidated hashing to Impl, Flip duet between primary and secondary, EVCacheInternalImpl
2 parents 86f5567 + 7b632f4 commit 7421dcb

File tree

13 files changed

+878
-501
lines changed

13 files changed

+878
-501
lines changed

‎evcache-client/build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ configurations {
1717
dependencies {
1818
compile project(':evcache-core')
1919
compile group:"net.spy", name:"spymemcached", version:"2.11.4"
20-
compile group:"com.netflix.evcache", name:"evcache-core", version:"5.+"
20+
2121
compile group:"com.google.code.findbugs", name:"annotations", version:"latest.release"
2222
compile group:"com.netflix.archaius", name:"archaius2-api", version:"latest.release"
2323
compile group:"com.netflix.archaius", name:"archaius2-core", version:"latest.release"

‎evcache-client/test/com/netflix/evcache/test/EVCacheTestDI.java

+206-21
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,32 @@
55

66
import java.util.Map;
77
import java.util.Properties;
8-
import java.util.concurrent.Future;
98

9+
import java.util.*;
10+
import java.util.concurrent.Future;
11+
import java.util.concurrent.TimeUnit;
12+
13+
import com.netflix.evcache.*;
14+
import com.netflix.evcache.operation.EVCacheItem;
15+
import com.netflix.evcache.operation.EVCacheItemMetaData;
16+
import com.netflix.evcache.pool.EVCacheClient;
17+
import com.netflix.evcache.pool.ServerGroup;
18+
import com.netflix.evcache.util.KeyHasher;
1019
import org.slf4j.Logger;
1120
import org.slf4j.LoggerFactory;
1221
import org.testng.annotations.Test;
1322

14-
import com.netflix.evcache.EVCache;
15-
import com.netflix.evcache.EVCacheGetOperationListener;
1623
import com.netflix.evcache.operation.EVCacheOperationFuture;
17-
1824
import rx.schedulers.Schedulers;
1925

26+
27+
import static org.testng.Assert.*;
28+
2029
public class EVCacheTestDI extends DIBase implements EVCacheGetOperationListener<String> {
2130
private static final Logger log = LoggerFactory.getLogger(EVCacheTestDI.class);
2231
private int loops = 1;
32+
private Map<String, String> propertiesToSet;
33+
private String appName = "EVCACHE_TEST";
2334

2435
public static void main(String args[]) {
2536
try {
@@ -31,31 +42,32 @@ public static void main(String args[]) {
3142
}
3243

3344
public EVCacheTestDI() {
45+
propertiesToSet = new HashMap<>();
46+
propertiesToSet.putIfAbsent(appName + ".us-east-1d.EVCacheClientPool.writeOnly", "false");
47+
propertiesToSet.putIfAbsent(appName + ".EVCacheClientPool.poolSize", "1");
48+
propertiesToSet.putIfAbsent(appName + ".ping.servers", "false");
49+
propertiesToSet.putIfAbsent(appName + ".cid.throw.exception", "true");
50+
propertiesToSet.putIfAbsent(appName + ".EVCacheClientPool.readTimeout", "500");
51+
propertiesToSet.putIfAbsent(appName + ".EVCacheClientPool.bulkReadTimeout", "500");
52+
propertiesToSet.putIfAbsent(appName + ".max.read.queue.length", "20");
53+
propertiesToSet.putIfAbsent("EVCacheClientPoolManager.log.apps", appName);
54+
propertiesToSet.putIfAbsent(appName + ".fallback.zone", "true");
55+
propertiesToSet.putIfAbsent(appName + ".enable.throttling", "false");
56+
propertiesToSet.putIfAbsent(appName + ".throttle.time", "0");
57+
propertiesToSet.putIfAbsent(appName + ".throttle.percent", "0");
58+
propertiesToSet.putIfAbsent(appName + ".log.operation", "1000");
59+
propertiesToSet.putIfAbsent(appName + ".EVCacheClientPool.validate.input.queue", "true");
3460
}
3561

3662
protected Properties getProps() {
3763
Properties props = super.getProps();
38-
props.setProperty("EVCACHE_CCS.us-east-1d.EVCacheClientPool.writeOnly", "false");
39-
props.setProperty("EVCACHE_CCS.EVCacheClientPool.poolSize", "1");
40-
props.setProperty("EVCACHE_CCS.ping.servers", "false");
41-
props.setProperty("EVCACHE_CCS.cid.throw.exception", "true");
42-
props.setProperty("EVCACHE_CCS.EVCacheClientPool.readTimeout", "500");
43-
props.setProperty("EVCACHE_CCS.EVCacheClientPool.bulkReadTimeout", "500");
44-
props.setProperty("EVCACHE_CCS.max.read.queue.length", "20");
45-
props.setProperty("EVCacheClientPoolManager.log.apps", "EVCACHE_CCS");
46-
props.setProperty("EVCACHE_CCS.fallback.zone", "true");
47-
props.setProperty("EVCACHE_CCS.enable.throttling", "false");
48-
props.setProperty("EVCACHE_CCS.throttle.time", "0");
49-
props.setProperty("EVCACHE_CCS.throttle.percent", "0");
50-
props.setProperty("EVCACHE_CCS.log.operation", "1000");
51-
props.setProperty("EVCACHE_CCS.EVCacheClientPool.validate.input.queue", "true");
52-
64+
propertiesToSet.entrySet().forEach(entry -> props.setProperty(entry.getKey(), entry.getValue()));
5365
return props;
5466
}
5567

5668
@Test
5769
public void testEVCache() {
58-
this.evCache = getNewBuilder().setAppName("EVCACHE_CCS").setCachePrefix("cid").enableRetry().build();
70+
this.evCache = getNewBuilder().setAppName(appName).setCachePrefix("cid").enableRetry().build();
5971
assertNotNull(evCache);
6072
}
6173

@@ -86,7 +98,7 @@ public void testKeySizeCheck() throws Exception {
8698
}
8799
assertTrue(exceptionThrown);
88100
}
89-
101+
90102
}
91103

92104
@Test(dependsOnMethods = { "testKeySizeCheck" })
@@ -241,6 +253,179 @@ public void testAppendOrAdd() throws Exception {
241253
}
242254
}
243255

256+
private void refreshEVCache() {
257+
setupEnv();
258+
testEVCache();
259+
}
260+
261+
@Test(dependsOnMethods = {"testAppendOrAdd"})
262+
public void functionalTestsWithAppLevelAndASGLevelHashingScenarios() throws Exception {
263+
refreshEVCache();
264+
265+
// no hashing
266+
assertFalse(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".hash.key", Boolean.class).orElse(false).get());
267+
doFunctionalTests(false);
268+
269+
// hashing at app level
270+
propertiesToSet.put(appName + ".hash.key", "true");
271+
refreshEVCache();
272+
assertTrue(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".hash.key", Boolean.class).orElse(false).get());
273+
doFunctionalTests(true);
274+
propertiesToSet.remove(appName + ".hash.key");
275+
276+
// hashing at app level due to auto hashing as a consequence of a large key
277+
propertiesToSet.put(appName + ".auto.hash.keys", "true");
278+
refreshEVCache();
279+
assertTrue(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".auto.hash.keys", Boolean.class).orElse(false).get());
280+
assertFalse(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".hash.key", Boolean.class).orElse(false).get());
281+
testWithLargeKey();
282+
// negative scenario
283+
propertiesToSet.remove(appName + ".auto.hash.keys");
284+
refreshEVCache();
285+
assertFalse(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".auto.hash.keys", Boolean.class).orElse(false).get());
286+
assertFalse(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".hash.key", Boolean.class).orElse(false).get());
287+
assertThrows(IllegalArgumentException.class, () -> {
288+
testWithLargeKey();
289+
});
290+
291+
// hashing at app level by choice AND different hashing at each asg
292+
Map<String, KeyHasher.HashingAlgorithm> hashingAlgorithmsByServerGroup = new HashMap<>();
293+
propertiesToSet.put(appName + ".hash.key", "true");
294+
refreshEVCache();
295+
assertTrue(manager.getEVCacheConfig().getPropertyRepository().get(appName + ".hash.key", Boolean.class).orElse(false).get());
296+
297+
// get server group names, to be used to configure the ASG level hashing properties
298+
Map<ServerGroup, List<EVCacheClient>> clientsByServerGroup = manager.getEVCacheClientPool(appName).getAllInstancesByServerGroup();
299+
int i = 0;
300+
for (ServerGroup serverGroup : clientsByServerGroup.keySet()) {
301+
KeyHasher.HashingAlgorithm hashingAlgorithm = KeyHasher.HashingAlgorithm.values()[i++ % KeyHasher.HashingAlgorithm.values().length];
302+
hashingAlgorithmsByServerGroup.put(serverGroup.getName(), hashingAlgorithm);
303+
propertiesToSet.put(serverGroup.getName() + ".hash.key", "true");
304+
propertiesToSet.put(serverGroup.getName() + ".hash.algo", hashingAlgorithm.name());
305+
}
306+
refreshEVCache();
307+
clientsByServerGroup = manager.getEVCacheClientPool(appName).getAllInstancesByServerGroup();
308+
// validate hashing properties of asgs
309+
for (ServerGroup serverGroup : clientsByServerGroup.keySet()) {
310+
assertEquals(clientsByServerGroup.get(serverGroup).get(0).getHashingAlgorithm(), hashingAlgorithmsByServerGroup.get(serverGroup.getName()));
311+
}
312+
doFunctionalTests(true);
313+
for (ServerGroup serverGroup : clientsByServerGroup.keySet()) {
314+
propertiesToSet.remove(serverGroup.getName());
315+
}
316+
}
317+
318+
private void testWithLargeKey() throws Exception {
319+
StringBuilder sb = new StringBuilder();
320+
for (int i= 0; i < 100; i++) {
321+
sb.append(Long.toString(System.currentTimeMillis()));
322+
}
323+
String key = sb.toString();
324+
String value = UUID.randomUUID().toString();
325+
326+
// set
327+
EVCacheLatch latch = evCache.set(key, value, EVCacheLatch.Policy.ALL);
328+
latch.await(1000, TimeUnit.MILLISECONDS);
329+
330+
// get
331+
assertEquals(evCache.get(key), value);
332+
}
333+
334+
private void doFunctionalTests(boolean isHashingEnabled) throws Exception {
335+
String key1 = Long.toString(System.currentTimeMillis());
336+
String value1 = UUID.randomUUID().toString();
337+
338+
// set
339+
EVCacheLatch latch = evCache.set(key1, value1, EVCacheLatch.Policy.ALL);
340+
latch.await(1000, TimeUnit.MILLISECONDS);
341+
342+
// get
343+
assertEquals(evCache.get(key1), value1);
344+
345+
// replace
346+
value1 = UUID.randomUUID().toString();
347+
latch = evCache.replace(key1, value1, EVCacheLatch.Policy.ALL);
348+
latch.await(1000, TimeUnit.MILLISECONDS);
349+
// get
350+
assertEquals(evCache.get(key1), value1);
351+
352+
// add a key
353+
String key2 = Long.toString(System.currentTimeMillis());
354+
String value2 = UUID.randomUUID().toString();
355+
latch = evCache.add(key2, value2, null, 1000, EVCacheLatch.Policy.ALL);
356+
latch.await(1000, TimeUnit.MILLISECONDS);
357+
// get
358+
assertEquals(evCache.get(key2), value2);
359+
360+
// appendoradd - append case
361+
String value3 = UUID.randomUUID().toString();
362+
if (isHashingEnabled) {
363+
assertThrows(EVCacheException.class, () -> {
364+
evCache.appendOrAdd(key2, value3, null, 1000, EVCacheLatch.Policy.ALL);
365+
});
366+
} else {
367+
latch = evCache.appendOrAdd(key2, value3, null, 1000, EVCacheLatch.Policy.ALL);
368+
latch.await(3000, TimeUnit.MILLISECONDS);
369+
assertEquals(evCache.get(key2), value2 + value3);
370+
}
371+
372+
// appendoradd - add case
373+
String key3 = Long.toString(System.currentTimeMillis());
374+
String value4 = UUID.randomUUID().toString();
375+
if (isHashingEnabled) {
376+
assertThrows(EVCacheException.class, () -> {
377+
evCache.appendOrAdd(key3, value4, null, 1000, EVCacheLatch.Policy.ALL);
378+
});
379+
} else {
380+
latch = evCache.appendOrAdd(key3, value4, null, 1000, EVCacheLatch.Policy.ALL);
381+
latch.await(3000, TimeUnit.MILLISECONDS);
382+
// get
383+
assertEquals(evCache.get(key3), value4);
384+
}
385+
386+
// append
387+
String value5 = UUID.randomUUID().toString();
388+
if (isHashingEnabled) {
389+
assertThrows(EVCacheException.class, () -> {
390+
evCache.append(key3, value5, 1000);
391+
});
392+
} else {
393+
Future<Boolean> futures[] = evCache.append(key3, value5, 1000);
394+
for (Future future : futures) {
395+
assertTrue((Boolean) future.get());
396+
}
397+
// get
398+
assertEquals(evCache.get(key3), value4 + value5);
399+
}
400+
401+
String key4 = Long.toString(System.currentTimeMillis());
402+
assertEquals(evCache.incr(key4, 1, 10, 1000), 10);
403+
assertEquals(evCache.incr(key4, 10, 10, 1000), 20);
404+
405+
// decr
406+
String key5 = Long.toString(System.currentTimeMillis());
407+
assertEquals(evCache.decr(key5, 1, 10, 1000), 10);
408+
assertEquals(evCache.decr(key5, 20, 10, 1000), 0);
409+
410+
// delete
411+
latch = evCache.delete(key1, EVCacheLatch.Policy.ALL);
412+
latch.await(1000, TimeUnit.MILLISECONDS);
413+
latch = evCache.delete(key2, EVCacheLatch.Policy.ALL);
414+
latch.await(1000, TimeUnit.MILLISECONDS);
415+
latch = evCache.delete(key3, EVCacheLatch.Policy.ALL);
416+
latch.await(1000, TimeUnit.MILLISECONDS);
417+
latch = evCache.delete(key4, EVCacheLatch.Policy.ALL);
418+
latch.await(1000, TimeUnit.MILLISECONDS);
419+
latch = evCache.delete(key5, EVCacheLatch.Policy.ALL);
420+
latch.await(1000, TimeUnit.MILLISECONDS);
421+
422+
assertNull(evCache.get(key1));
423+
assertNull(evCache.get(key2));
424+
assertNull(evCache.get(key3));
425+
assertNull(evCache.get(key4));
426+
assertNull(evCache.get(key5));
427+
}
428+
244429
public void testAll() {
245430
try {
246431
setupEnv();

‎evcache-core/src/main/java/com/netflix/evcache/EVCache.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,10 @@ public Builder customizeWith(final Customizer customizer) {
13921392
return this;
13931393
}
13941394

1395+
protected EVCache newImpl(String appName, String cachePrefix, int ttl, Transcoder<?> transcoder, boolean serverGroupRetry, boolean enableExceptionThrowing, EVCacheClientPoolManager poolManager) {
1396+
return new EVCacheImpl(appName, cachePrefix, ttl, transcoder, serverGroupRetry, enableExceptionThrowing, poolManager);
1397+
}
1398+
13951399
/**
13961400
* Returns a newly created {@code EVCache} based on the contents of the
13971401
* {@code Builder}.
@@ -1417,8 +1421,7 @@ public EVCache build() {
14171421

14181422
customize();
14191423

1420-
return new EVCacheImpl(
1421-
_appName, _cachePrefix, _ttl, _transcoder, _serverGroupRetry, _enableExceptionThrowing, _poolManager);
1424+
return newImpl(_appName, _cachePrefix, _ttl, _transcoder, _serverGroupRetry, _enableExceptionThrowing, _poolManager);
14221425
}
14231426
}
14241427
}

0 commit comments

Comments
 (0)