Merge pull request #95 from Netflix/feature/base85

Added support for Ascii85 encoding

Author: sumanth-pasupuleti

evcache-core/build.gradle

@@ -38,6 +38,7 @@ dependencies {
         compile group:"org.apache.httpcomponents",    name:"httpclient",                       version:"latest.release"
         compile group:"joda-time",                    name:"joda-time",                        version:"latest.release"
         compile group:"javax.annotation",             name:"javax.annotation-api",             version:"latest.release"
+        compile group:"com.github.fzakaria",          name:"ascii85",                          version:"latest.release"
 
         testCompile group:"org.testng",               name:"testng",                           version:"latest.release"
         testCompile group:"com.beust",                name:"jcommander",                       version:"1.72"

evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java

@@ -165,7 +165,7 @@ private EVCacheLatch addOrSet(boolean replaceItem, String key, CachedData value,
             // identify that evcache client whose primary node is the destination ip for the key being processed
             evCacheClients = evCacheClients.stream().filter(client ->
                     destinationIps.contains(((InetSocketAddress) client.getNodeLocator()
-                        .getPrimary(getEVCacheKey(key).getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength()))
+                        .getPrimary(getEVCacheKey(key).getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder()))
                         .getSocketAddress()).getAddress().getHostAddress())
             ).collect(Collectors.toList());
         }

evcache-core/src/main/java/com/netflix/evcache/EVCacheInternalImpl.java

@@ -19,10 +19,15 @@ public class EVCacheKey {
     // like max.hash.length. So changing max.hash.length alone would not necessarily trigger hash recalculation, but
     // one would have to change the hashing algorithm in order to having hashing properties taken into account.
     // This is to make such a hashing property change very obvious and not subtle.
-    private final Map<HashingAlgorithm, String> hashedKeysByAlgorithm;
-    private final Map<HashingAlgorithm, String> hashedKeysByAlgorithmForDuet;
+    private final Map<String, String> hashedKeysByAlgorithm;
+    private final Map<String, String> hashedKeysByAlgorithmForDuet;
+    private final String encoder;
 
     public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgorithm hashingAlgorithmAtAppLevel, Property<Boolean> shouldEncodeHashKeyAtAppLevel, Property<Integer> maxDigestBytesAtAppLevel, Property<Integer> maxHashLengthAtAppLevel) {
+        this(appName, key, canonicalKey, hashingAlgorithmAtAppLevel, shouldEncodeHashKeyAtAppLevel, maxDigestBytesAtAppLevel, maxHashLengthAtAppLevel, null);
+    }
+
+    public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgorithm hashingAlgorithmAtAppLevel, Property<Boolean> shouldEncodeHashKeyAtAppLevel, Property<Integer> maxDigestBytesAtAppLevel, Property<Integer> maxHashLengthAtAppLevel, String encoder) {
         super();
         this.appName = appName;
         this.key = key;
@@ -31,6 +36,7 @@ public EVCacheKey(String appName, String key, String canonicalKey, HashingAlgori
         this.shouldEncodeHashKeyAtAppLevel = shouldEncodeHashKeyAtAppLevel;
         this.maxDigestBytesAtAppLevel = maxDigestBytesAtAppLevel;
         this.maxHashLengthAtAppLevel = maxHashLengthAtAppLevel;
+        this.encoder = encoder;
         hashedKeysByAlgorithm = new HashMap<>();
         hashedKeysByAlgorithmForDuet = new HashMap<>();
     }
@@ -59,11 +65,11 @@ private String getCanonicalKeyForDuet() {
 
     @Deprecated
     public String getHashKey() {
-        return getHashKey(hashingAlgorithmAtAppLevel, null == shouldEncodeHashKeyAtAppLevel ? null : shouldEncodeHashKeyAtAppLevel.get(), null == maxDigestBytesAtAppLevel ? null : maxDigestBytesAtAppLevel.get(), null == maxHashLengthAtAppLevel ? null : maxHashLengthAtAppLevel.get());
+        return getHashKey(hashingAlgorithmAtAppLevel, null == shouldEncodeHashKeyAtAppLevel ? null : shouldEncodeHashKeyAtAppLevel.get(), null == maxDigestBytesAtAppLevel ? null : maxDigestBytesAtAppLevel.get(), null == maxHashLengthAtAppLevel ? null : maxHashLengthAtAppLevel.get(), encoder);
     }
 
     // overlays app level hashing and client level hashing
-    public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
+    public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String baseEnoder) {
         if (hashingAlgorithm == HashingAlgorithm.NO_HASHING) {
             return null;
         }
@@ -83,36 +89,52 @@ public String getHashKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Bool
         if (null == maxHashLength) {
             maxHashLength = this.maxHashLengthAtAppLevel.get();
         }
+        
+        if(null == baseEnoder) {
+            baseEnoder = encoder;
+        }
 
-        return isDuet ? getHashKeyForDuet(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength) : getHashKey(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength);
+        return isDuet ? getHashKeyForDuet(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder) : getHashKey(hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder);
     }
 
     // overlays app level hashing algorithm and client level hashing algorithm
-    public String getDerivedKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
+    public String getDerivedKey(boolean isDuet, HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String baseEnoder) {
         // this overlay of hashingAlgorithm helps determine if there at all needs to be hashing performed, otherwise, will return canonical key
         if (null == hashingAlgorithm) {
             hashingAlgorithm = hashingAlgorithmAtAppLevel;
         }
 
-        return null == hashingAlgorithm || hashingAlgorithm == HashingAlgorithm.NO_HASHING ? getCanonicalKey(isDuet) : getHashKey(isDuet, hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength);
+        return null == hashingAlgorithm || hashingAlgorithm == HashingAlgorithm.NO_HASHING ? getCanonicalKey(isDuet) : getHashKey(isDuet, hashingAlgorithm, shouldEncodeHashKey, maxDigestBytes, maxHashLength, baseEnoder);
     }
 
-    private String getHashKey(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
+    private String getHashKey(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String encoder) {
         if (null == hashingAlgorithm) {
             return null;
         }
 
+        final String key = hashingAlgorithm.toString()+ maxDigestBytes != null ? maxDigestBytes.toString() : "-" + maxHashLength != null ? maxHashLength.toString() : "-" + encoder != null ? encoder : "-";
+        String val = hashedKeysByAlgorithm.get(key);
+        if(val == null) {
+            val = KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), hashingAlgorithm, maxDigestBytes, maxHashLength, encoder);
+            hashedKeysByAlgorithm.put(key , val);
+        }
         // TODO: Once the issue around passing hashedKey in bytes[] is figured, we will start using (nullable) shouldEncodeHashKey, and call KeyHasher.getHashedKeyInBytes() accordingly
-        return hashedKeysByAlgorithm.computeIfAbsent(hashingAlgorithm, ha -> KeyHasher.getHashedKeyEncoded(canonicalKey, ha, maxDigestBytes, maxHashLength));
+        return val;
     }
 
-    private String getHashKeyForDuet(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength) {
+    private String getHashKeyForDuet(HashingAlgorithm hashingAlgorithm, Boolean shouldEncodeHashKey, Integer maxDigestBytes, Integer maxHashLength, String encoder) {
         if (null == hashingAlgorithm) {
             return null;
         }
 
+        final String key = hashingAlgorithm.toString()+ maxDigestBytes != null ? maxDigestBytes.toString() : "-" + maxHashLength != null ? maxHashLength.toString() : "-" + encoder != null ? encoder : "-";
+        String val = hashedKeysByAlgorithmForDuet.get(key);
+        if(val == null) {
+            val = KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), hashingAlgorithm, maxDigestBytes, maxHashLength, encoder);
+            hashedKeysByAlgorithmForDuet.put(key , val);
+        }
         // TODO: Once the issue around passing hashedKey in bytes[] is figured, we will start using (nullable) shouldEncodeHashKey, and call KeyHasher.getHashedKeyInBytes() accordingly
-        return hashedKeysByAlgorithmForDuet.computeIfAbsent(hashingAlgorithm, ha -> KeyHasher.getHashedKeyEncoded(getCanonicalKeyForDuet(), ha, maxDigestBytes, maxHashLength));
+        return val;
     }
 
     @Override

evcache-core/src/main/java/com/netflix/evcache/EVCacheKey.java

@@ -170,7 +170,7 @@ public Collection<String> getCanonicalKeys() {
     public Collection<MemcachedNode> getMemcachedNode(EVCacheKey evckey) {
         final Collection<MemcachedNode> nodeList = new ArrayList<MemcachedNode>(clients.size());
         for(EVCacheClient client : clients) {
-            String key = evckey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
+            String key = evckey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder());
             nodeList.add(client.getNodeLocator().getPrimary(key));
         }
         return nodeList;

evcache-core/src/main/java/com/netflix/evcache/event/EVCacheEvent.java

@@ -92,6 +92,8 @@ public class EVCacheClient {
     private final Property<Integer> maxDigestBytes;
     private final Property<Integer> maxHashLength;
     private final Property<Integer> chunkSize, writeBlock;
+    private final Property<String> encoderBase;
+
     private final ChunkTranscoder chunkingTranscoder;
     private final SerializingTranscoder decodingTranscoder;
     private static final int SPECIAL_BYTEARRAY = (8 << 8);
@@ -153,6 +155,7 @@ public class EVCacheClient {
         this.shouldEncodeHashKey = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".hash.encode", Boolean.class).orElse(null);
         this.maxDigestBytes = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".max.digest.bytes", Integer.class).orElse(null);
         this.maxHashLength = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".max.hash.length", Integer.class).orElse(null);
+        this.encoderBase = EVCacheConfig.getInstance().getPropertyRepository().get(this.serverGroup.getName() + ".hash.encoder", String.class).orElse("base64");
         ping();
     }
 
@@ -176,6 +179,10 @@ public Boolean shouldEncodeHashKey() {
         return this.shouldEncodeHashKey.get();
     }
 
+    public String getBaseEncoder() {
+        return this.encoderBase.get();
+    }
+
     public Integer getMaxDigestBytes() {
         return this.maxDigestBytes.get();
     }

evcache-core/src/main/java/com/netflix/evcache/pool/EVCacheClient.java

@@ -38,7 +38,7 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
         Boolean firstStatus = null;
         for (EVCacheClient client : clients) {
             CachedData cd1;
-            if (evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength()) != null) {
+            if (evcKey.getHashKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder()) != null) {
                 if(cdHashed == null) {
                     final EVCacheValue val = new EVCacheValue(evcKey.getCanonicalKey(client.isDuetClient()), cd.getData(), cd.getFlags(), timeToLive, System.currentTimeMillis());
                     cdHashed = evcacheValueTranscoder.encode(val);
@@ -47,7 +47,7 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
             } else {
             	cd1 = cd;
             }
-            String key = evcKey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength());
+            String key = evcKey.getDerivedKey(client.isDuetClient(), client.getHashingAlgorithm(), client.shouldEncodeHashKey(), client.getMaxDigestBytes(), client.getMaxHashLength(), client.getBaseEncoder());
             final Future<Boolean> f = client.add(key, timeToLive, cd1, latch);
             if (log.isDebugEnabled()) log.debug("ADD : Op Submitted : APP " + _appName + ", key " + key + "; future : " + f + "; client : " + client);
             if(fixMissing) {
@@ -71,12 +71,12 @@ public EVCacheLatch add(EVCacheKey evcKey, final CachedData cd, Transcoder evcac
     private EVCacheLatch fixup(EVCacheClient sourceClient, EVCacheClient[] destClients, EVCacheKey evcKey, int timeToLive, Policy policy) {
         final EVCacheLatchImpl latch = new EVCacheLatchImpl(policy, destClients.length, _appName);
         try {
-            final CachedData readData = sourceClient.get(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength()), ct, false, false);
+            final CachedData readData = sourceClient.get(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength(), sourceClient.getBaseEncoder()), ct, false, false);
 
             if(readData != null) {
-                sourceClient.touch(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength()), timeToLive);
+                sourceClient.touch(evcKey.getDerivedKey(sourceClient.isDuetClient(), sourceClient.getHashingAlgorithm(), sourceClient.shouldEncodeHashKey(), sourceClient.getMaxDigestBytes(), sourceClient.getMaxHashLength(), sourceClient.getBaseEncoder()), timeToLive);
                 for(EVCacheClient destClient : destClients) {
-                    destClient.set(evcKey.getDerivedKey(destClient.isDuetClient(), destClient.getHashingAlgorithm(), destClient.shouldEncodeHashKey(), destClient.getMaxDigestBytes(), destClient.getMaxHashLength()), readData, timeToLive, latch);
+                    destClient.set(evcKey.getDerivedKey(destClient.isDuetClient(), destClient.getHashingAlgorithm(), destClient.shouldEncodeHashKey(), destClient.getMaxDigestBytes(), destClient.getMaxHashLength(), destClient.getBaseEncoder()), readData, timeToLive, latch);
                 }
             }
             latch.await(_operationTimeout, TimeUnit.MILLISECONDS);