Merge pull request #537 from salesforce/W-13497368-final

W-13497368

Author: or-koren

timbermill-java/timbermill-local/src/main/java/com/datorama/oss/timbermill/common/cache/CacheConfig.java

@@ -5,14 +5,23 @@
 public class CacheConfig {
     private RedisService redisService;
     private int cacheRedisTtlInSeconds;
+    private int orphansCacheRedisTtlInSeconds;
+    private int eventsCacheRedisTtlInSeconds;
     private long maximumTasksCacheWeight;
     private long maximumOrphansCacheWeight;
 
     public CacheConfig(RedisService redisService, int cacheRedisTtlInSeconds, long maximumTasksCacheWeight, long maximumOrphansCacheWeight) {
+        this(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds);
+    }
+    public CacheConfig(RedisService redisService, int cacheRedisTtlInSeconds, long maximumTasksCacheWeight, long maximumOrphansCacheWeight, int orphansCacheRedisTtlInSeconds,
+                       int eventsCacheRedisTtlInSeconds) {
         this.redisService = redisService;
         this.cacheRedisTtlInSeconds = cacheRedisTtlInSeconds;
         this.maximumTasksCacheWeight = maximumTasksCacheWeight;
         this.maximumOrphansCacheWeight = maximumOrphansCacheWeight;
+        this.eventsCacheRedisTtlInSeconds = eventsCacheRedisTtlInSeconds;
+        this.orphansCacheRedisTtlInSeconds = orphansCacheRedisTtlInSeconds;
+
     }
 
     RedisService getRedisService() {
@@ -31,4 +40,12 @@ long getMaximumOrphansCacheWeight() {
         return maximumOrphansCacheWeight;
     }
 
+    public int getOrphansCacheRedisTtlInSeconds() {
+        return orphansCacheRedisTtlInSeconds;
+    }
+
+    public int getEventsCacheRedisTtlInSeconds() {
+        return eventsCacheRedisTtlInSeconds;
+    }
+
 }

timbermill-java/timbermill-local/src/main/java/com/datorama/oss/timbermill/common/cache/CacheHandlerUtil.java

@@ -3,7 +3,7 @@
 public class CacheHandlerUtil {
     public static AbstractCacheHandler getCacheHandler(String strategy, CacheConfig cacheParams) {
         if (strategy.compareToIgnoreCase("redis") == 0){
-            return new RedisCacheHandler(cacheParams.getRedisService(), cacheParams.getCacheRedisTtlInSeconds());
+            return new RedisCacheHandler(cacheParams.getRedisService(), cacheParams.getCacheRedisTtlInSeconds(), cacheParams.getOrphansCacheRedisTtlInSeconds(), cacheParams.getEventsCacheRedisTtlInSeconds());
         }
         else {
             return new LocalCacheHandler(cacheParams.getMaximumTasksCacheWeight(), cacheParams.getMaximumOrphansCacheWeight());

timbermill-java/timbermill-local/src/main/java/com/datorama/oss/timbermill/common/cache/RedisCacheHandler.java

@@ -23,20 +23,24 @@ public class RedisCacheHandler extends AbstractCacheHandler {
     private JedisLock lock;
     private final RedisService redisService;
     private final int redisTtlInSeconds;
-    
-    private int orphanRedisTTLRedisTTLInSeconds;
-    private int cacheRedisTTLRedisTTLInSeconds;
 
+    private int cacheOrphansRedisTtlInSeconds;
+    private int cacheEventsRedisTtlInSeconds;
 
     RedisCacheHandler(RedisService redisService, int cacheRedisTtlInSeconds) {
+        this(redisService, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds, cacheRedisTtlInSeconds);
+    }
+
+    RedisCacheHandler(RedisService redisService, int cacheRedisTtlInSeconds, int cacheOrphansRedisTtlInSeconds, int cacheEventsRedisTtlInSeconds) {
         if (redisService == null){
             throw new RuntimeException("Redis cache used but no redis host defined");
         }
         this.redisService = redisService;
         this.redisTtlInSeconds = cacheRedisTtlInSeconds;
-        orphanRedisTTLRedisTTLInSeconds = System.getenv("ORPHANS_REDIS_TTL_SECONDS") != null ? Integer.parseInt(System.getenv("ORPHANS_REDIS_TTL_SECONDS")) : cacheRedisTtlInSeconds;
-        cacheRedisTTLRedisTTLInSeconds = System.getenv("CACHE_REDIS_TTL_SECONDS") != null ? Integer.parseInt(System.getenv("CACHE_REDIS_TTL_SECONDS")) : cacheRedisTtlInSeconds;
-        LOG.info("Initializing Timbermill Redis Cache with orphanRedisTTLRedisTTLInSeconds {} seconds and cacheRedisTTLRedisTTLInSeconds {}", orphanRedisTTLRedisTTLInSeconds, cacheRedisTTLRedisTTLInSeconds);
+        this.cacheEventsRedisTtlInSeconds = cacheEventsRedisTtlInSeconds;
+        this.cacheOrphansRedisTtlInSeconds = cacheOrphansRedisTtlInSeconds;
+        LOG.info("Initializing Timbermill Redis Cache with cacheEventsRedisTtlInSeconds: {}, cacheOrphansRedisTtlInSeconds: {}, cacheRedisTtlInSeconds:{} ", cacheEventsRedisTtlInSeconds,
+                cacheOrphansRedisTtlInSeconds, cacheRedisTtlInSeconds);
     }
 
     @Override
@@ -59,8 +63,7 @@ public void pushToOrphanCache(Map<String, List<String>> orphansMap) {
             String orphanCacheKey = ORPHAN_PREFIX + entry.getKey();
             newOrphansMap.put(orphanCacheKey, entry.getValue());
         }
-
-        if (!redisService.pushToRedis(newOrphansMap, orphanRedisTTLRedisTTLInSeconds)){
+        if (!redisService.pushToRedis(newOrphansMap, cacheOrphansRedisTtlInSeconds)){
             LOG.error("Failed to push some ids to Redis orphans cache.");
             KamonConstants.ORPHAN_CACHE_FAILED_COUNTER.withoutTags().increment();
         }
@@ -73,7 +76,7 @@ public Map<String, LocalTask> getFromTasksCache(Collection<String> idsList) {
 
     @Override
     public void pushToTasksCache(Map<String, LocalTask> idsToMap) {
-        boolean allPushed = redisService.pushToRedis(idsToMap, cacheRedisTTLRedisTTLInSeconds);
+        boolean allPushed = redisService.pushToRedis(idsToMap, cacheEventsRedisTtlInSeconds);
         if (!allPushed){
             LOG.error("Failed to push some ids to Redis tasks cache.");
         }

timbermill-java/timbermill-server/src/main/java/com/datorama/timbermill/server/service/TimbermillService.java

@@ -96,6 +96,8 @@ public TimbermillService(@Value("${INDEX_BULK_SIZE:200000}") Integer indexBulkSi
 							 @Value("${MAXIMUM_ORPHANS_CACHE_WEIGHT:1000000000}") long maximumOrphansCacheWeight,
 							 @Value("${CACHE_STRATEGY:}") String cacheStrategy,
 							 @Value("${CACHE_TTL_IN_SECONDS:604800}") int cacheRedisTtlInSeconds,
+							 @Value("${ORPHAN_CACHE_TTL_IN_SECONDS:86400}") int orphansCacheRedisTtlInSeconds, //One day
+							 @Value("${EVENTS_CACHE_TTL_IN_SECONDS:86400}") int eventsCacheRedisTtlInSeconds, //One day
 							 @Value("${REDIS_MAX_MEMORY:}") String redisMaxMemory,
 							 @Value("${REDIS_MAX_MEMORY_POLICY:}") String redisMaxMemoryPolicy,
 							 @Value("${REDIS_HOST:}") String redisHost,
@@ -137,7 +139,7 @@ public TimbermillService(@Value("${INDEX_BULK_SIZE:200000}") Integer indexBulkSi
 				elasticPassword, maxIndexAge, maxIndexSizeInGB, maxIndexDocs, numOfElasticSearchActionsTries, maxBulkIndexFetches, searchMaxSize, persistenceHandler, numberOfShards, numberOfReplicas,
 				maxTotalFields, null, scrollLimitation, scrollTimeoutSeconds, fetchByIdsPartitions, expiredMaxIndicesToDeleteInParallel);
 
-		CacheConfig cacheParams = new CacheConfig(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight);
+		CacheConfig cacheParams = new CacheConfig(redisService, cacheRedisTtlInSeconds, maximumTasksCacheWeight, maximumOrphansCacheWeight, orphansCacheRedisTtlInSeconds, eventsCacheRedisTtlInSeconds);
 		AbstractCacheHandler cacheHandler = CacheHandlerUtil.getCacheHandler(cacheStrategy, cacheParams);
 		this.eventsMaxElement = eventsMaxElement;
 		taskIndexer = new TaskIndexer(pluginsJson, daysRotation, es, timbermillVersion, cacheHandler);