[server][dvc] Fix use-after-free in RMD block cache metrics callbacks (linkedin#2742)

Fix JVM SIGSEGV crashes caused by RMD block cache async gauge callbacks  
dereferencing a freed RocksDB Cache JNI handle after factory shutdown. The  
callbacks previously captured `rmdCache` directly and could still run on OTel  
or Tehuti collection threads after native cache memory had already been freed.  

Store the cache in a volatile field and have callbacks read it into a local  
variable before the null check and JNI call. Also null the reference in  
`closeRMDBlockCache()` and invoke it from  
`RocksDBStorageEngineFactory.close()` before `sharedRMDCache.close()`, so  
metrics callbacks observe null before native memory is released.

Author: sushantmane

clients/da-vinci-client/src/main/java/com/linkedin/davinci/stats/RocksDBMemoryStats.java

@@ -189,23 +189,50 @@ public void deregisterPartition(String partitionName) {
     }
   }
 
+  /**
+   * Volatile reference cleared by {@link #closeRMDBlockCache()} before the native Cache is freed.
+   * Callbacks read this reference into a local variable — if null, the cache is closed and they
+   * return 0. The local variable pins the reference on the stack, eliminating any TOCTOU race
+   * between the null check and the JNI call without requiring locks.
+   */
+  private volatile Cache rmdCache;
+
   public void setRMDBlockCache(Cache rmdCache, long rmdCacheCapacity) {
     if (!rmdBlockCacheRegistered.compareAndSet(false, true)) {
       LOGGER.warn("setRMDBlockCache called more than once; ignoring duplicate registration");
       return;
     }
+    this.rmdCache = rmdCache;
     registerAsyncGauge(
         RocksDBMemoryOtelMetricEntity.RMD_BLOCK_CACHE_CAPACITY,
         TehutiMetricName.RMD_BLOCK_CACHE_CAPACITY,
         () -> rmdCacheCapacity);
     registerAsyncGauge(
         RocksDBMemoryOtelMetricEntity.RMD_BLOCK_CACHE_USAGE,
         TehutiMetricName.RMD_BLOCK_CACHE_USAGE,
-        rmdCache::getUsage);
+        this::getRMDCacheUsage);
     registerAsyncGauge(
         RocksDBMemoryOtelMetricEntity.RMD_BLOCK_CACHE_PINNED_USAGE,
         TehutiMetricName.RMD_BLOCK_CACHE_PINNED_USAGE,
-        rmdCache::getPinnedUsage);
+        this::getRMDCachePinnedUsage);
+  }
+
+  /**
+   * Must be called before closing the native Cache object to prevent use-after-free.
+   * Nulls out the volatile reference so in-flight and future callbacks return 0.
+   */
+  public void closeRMDBlockCache() {
+    this.rmdCache = null;
+  }
+
+  private long getRMDCacheUsage() {
+    Cache cache = this.rmdCache;
+    return cache != null ? cache.getUsage() : 0L;
+  }
+
+  private long getRMDCachePinnedUsage() {
+    Cache cache = this.rmdCache;
+    return cache != null ? cache.getPinnedUsage() : 0L;
   }
 
   /** Registers a joint Tehuti+OTel async gauge metric. */

clients/da-vinci-client/src/main/java/com/linkedin/davinci/store/rocksdb/RocksDBStorageEngineFactory.java

@@ -303,6 +303,11 @@ public synchronized void close() {
       storageEngine.close();
     });
     storageEngineMap.clear();
+    // Null out the RMD cache reference in metrics BEFORE closing the native object.
+    // This prevents OTel/Tehuti async gauge callbacks from dereferencing a freed JNI handle.
+    if (rocksDBMemoryStats != null) {
+      rocksDBMemoryStats.closeRMDBlockCache();
+    }
     sharedCache.close();
     if (sharedRMDCache != null) {
       sharedRMDCache.close();