Add basic inode cache

core/common/src/main/java/alluxio/conf/PropertyKey.java

@@ -2676,6 +2676,16 @@ public String toString() {
           .setConsistencyCheckLevel(ConsistencyCheckLevel.ENFORCE)
           .setScope(Scope.MASTER)
           .build();
+  public static final PropertyKey MASTER_METASTORE_INODE_CACHE_BASIC =
+      booleanBuilder(Name.MASTER_METASTORE_INODE_CACHE_BASIC)
+          .setDefaultValue(false)
+          .setDescription("If true a basic cache is used that only caches inodes "
+              + "with edges being accessed from RocksDB directly. "
+              + "This cache is recommended when using a master with limited memory "
+              + "or when individual directories contain large numbers of files.")
+          .setConsistencyCheckLevel(ConsistencyCheckLevel.ENFORCE)
+          .setScope(Scope.MASTER)
+          .build();
   // 2k bytes per inode cache key and * 2 for the existence of edge cache and some leeway
   public static final PropertyKey MASTER_METASTORE_INODE_CACHE_MAX_SIZE =
       intBuilder(Name.MASTER_METASTORE_INODE_CACHE_MAX_SIZE)
@@ -8169,6 +8179,8 @@ public static final class Name {
         "alluxio.master.metastore.inode.cache.high.water.mark.ratio";
     public static final String MASTER_METASTORE_INODE_CACHE_LOW_WATER_MARK_RATIO =
         "alluxio.master.metastore.inode.cache.low.water.mark.ratio";
+    public static final String MASTER_METASTORE_INODE_CACHE_BASIC =
+        "alluxio.master.metastore.inode.cache.basic";
     public static final String MASTER_METASTORE_INODE_CACHE_MAX_SIZE =
         "alluxio.master.metastore.inode.cache.max.size";
     public static final String MASTER_METASTORE_INODE_ITERATION_CRAWLER_COUNT =

core/server/master/pom.xml

@@ -30,6 +30,11 @@
 
   <dependencies>
     <!-- External dependencies -->
+    <dependency>
+      <groupId>com.github.ben-manes.caffeine</groupId>
+      <artifactId>caffeine</artifactId>
+      <version>2.9.3</version>
+    </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>

core/server/master/src/main/java/alluxio/master/MasterUtils.java

@@ -17,6 +17,7 @@
 import alluxio.master.metastore.BlockMetaStore;
 import alluxio.master.metastore.InodeStore;
 import alluxio.master.metastore.MetastoreType;
+import alluxio.master.metastore.caching.BasicInodeCache;
 import alluxio.master.metastore.caching.CachingInodeStore;
 import alluxio.master.metastore.heap.HeapBlockMetaStore;
 import alluxio.master.metastore.heap.HeapInodeStore;
@@ -90,7 +91,11 @@ public static InodeStore.Factory getInodeStoreFactory(String baseDir) {
         if (Configuration.getInt(PropertyKey.MASTER_METASTORE_INODE_CACHE_MAX_SIZE) == 0) {
           return lockManager -> new RocksInodeStore(baseDir);
         } else {
-          return lockManager -> new CachingInodeStore(new RocksInodeStore(baseDir), lockManager);
+          if (Configuration.getBoolean(PropertyKey.MASTER_METASTORE_INODE_CACHE_BASIC)) {
+            return lockManager -> new BasicInodeCache(baseDir);
+          } else {
+            return lockManager -> new CachingInodeStore(new RocksInodeStore(baseDir), lockManager);
+          }
         }
       default:
         throw new IllegalStateException("Unknown metastore type: " + type);

core/server/master/src/main/java/alluxio/master/metastore/caching/BasicInodeCache.java

@@ -0,0 +1,74 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.master.metastore.caching;
+
+import alluxio.conf.Configuration;
+import alluxio.conf.PropertyKey;
+import alluxio.master.file.meta.MutableInode;
+import alluxio.master.metastore.ReadOption;
+import alluxio.master.metastore.rocks.RocksInodeStore;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.base.Preconditions;
+
+import java.util.Optional;
+
+/**
+ * A cache that only caches inodes.
+ */
+public class BasicInodeCache extends RocksInodeStore {
+
+  com.github.benmanes.caffeine.cache.Cache<Long, MutableInode<?>> mCache;
+
+  /**
+   * Creates and initializes a rocks block store.
+   *
+   * @param baseDir the base directory in which to store inode metadata
+   */
+  public BasicInodeCache(String baseDir) {
+    super(baseDir);
+
+    int maxSize = Configuration.getInt(PropertyKey.MASTER_METASTORE_INODE_CACHE_MAX_SIZE);
+    Preconditions.checkState(maxSize > 0,
+        "Maximum cache size %s must be positive, but is set to %s",
+        PropertyKey.MASTER_METASTORE_INODE_CACHE_MAX_SIZE.getName(), maxSize);
+
+    mCache = Caffeine.newBuilder().initialCapacity(maxSize).maximumSize(maxSize).build();
+  }
+
+  @Override
+  public Optional<MutableInode<?>> getMutable(long inodeId, ReadOption option) {
+    return Optional.ofNullable(mCache.asMap().computeIfAbsent(inodeId, id ->
+      BasicInodeCache.super.getMutable(inodeId, option).orElse(null)));
+  }
+
+  @Override
+  public void remove(Long inodeId) {
+    mCache.asMap().compute(inodeId, (id, inode) -> {
+      BasicInodeCache.super.remove(inodeId);
+      return null;
+    });
+  }
+
+  @Override
+  public void writeInode(MutableInode<?> inode) {
+    mCache.asMap().compute(inode.getId(), (id, oldInode) -> {
+      BasicInodeCache.super.writeInode(inode);
+      return inode;
+    });
+  }
+
+  @Override
+  public boolean supportsBatchWrite() {
+    return false;
+  }
+}

core/server/master/src/test/java/alluxio/master/file/FileSystemMasterTestBase.java

@@ -50,6 +50,7 @@
 import alluxio.master.journal.JournalType;
 import alluxio.master.metastore.InodeStore;
 import alluxio.master.metastore.ReadOnlyInodeStore;
+import alluxio.master.metastore.caching.BasicInodeCache;
 import alluxio.master.metastore.caching.CachingInodeStore;
 import alluxio.master.metastore.heap.HeapBlockMetaStore;
 import alluxio.master.metastore.heap.HeapInodeStore;
@@ -172,6 +173,7 @@ public static Iterable<InodeStore.Factory> parameters() throws Exception {
     return Arrays.asList(
         lockManager -> new HeapInodeStore(),
         lockManager -> new RocksInodeStore(dir),
+        lockManager -> new BasicInodeCache(dir),
         lockManager -> new CachingInodeStore(new RocksInodeStore(dir), lockManager));
   }
 

core/server/master/src/test/java/alluxio/master/file/meta/InodeTreeTest.java

@@ -45,6 +45,7 @@
 import alluxio.master.file.meta.options.MountInfo;
 import alluxio.master.journal.NoopJournalContext;
 import alluxio.master.metastore.InodeStore;
+import alluxio.master.metastore.caching.BasicInodeCache;
 import alluxio.master.metastore.caching.CachingInodeStore;
 import alluxio.master.metastore.heap.HeapInodeStore;
 import alluxio.master.metastore.rocks.RocksInodeStore;
@@ -125,6 +126,7 @@ public static Iterable<Supplier<InodeStore>> parameters() throws Exception {
         () -> new CachingInodeStore(new RocksInodeStore(dir), new InodeLockManager()),
         () -> new CachingInodeStore(new HeapInodeStore(), new InodeLockManager()),
         HeapInodeStore::new,
+        () -> new BasicInodeCache(dir),
         () -> new RocksInodeStore(dir));
   }
 

core/server/master/src/test/java/alluxio/master/metastore/InodeStoreTest.java

@@ -33,6 +33,7 @@
 import alluxio.master.file.meta.MutableInodeDirectory;
 import alluxio.master.file.meta.MutableInodeFile;
 import alluxio.master.metastore.InodeStore.WriteBatch;
+import alluxio.master.metastore.caching.BasicInodeCache;
 import alluxio.master.metastore.caching.CachingInodeStore;
 import alluxio.master.metastore.heap.HeapInodeStore;
 import alluxio.master.metastore.rocks.RocksInodeStore;
@@ -75,6 +76,7 @@ public static Iterable<Function<InodeLockManager, InodeStore>> parameters() thro
     return Arrays.asList(
         lockManager -> new HeapInodeStore(),
         lockManager -> new RocksInodeStore(sDir),
+        lockManager -> new BasicInodeCache(sDir),
         lockManager -> new CachingInodeStore(new RocksInodeStore(sDir), lockManager));
   }
 
@@ -259,6 +261,29 @@ public void repeatedAddRemoveAndList() {
     assertEquals(1, CloseableIterator.size(mStore.getChildren(mRoot)));
   }
 
+  @Test
+  public void overfullCacheRemove() {
+    writeInode(mRoot);
+    List<Long> fileIds = new ArrayList<>();
+    List<String> fileNames = new ArrayList<>();
+    long numFiles = CACHE_SIZE * 2;
+    // Create double the cache size files
+    for (int i = 0; i < numFiles; i++) {
+      MutableInodeFile file = inodeFile(i + 1000, 0, "file" + i);
+      fileIds.add(file.getId());
+      fileNames.add(file.getName());
+      writeInode(file);
+      writeEdge(mRoot, file);
+    }
+    // delete all the files
+    for (int i = 0; i < numFiles; i++) {
+      mStore.remove(fileIds.get(i));
+      mStore.removeChild(mRoot.getId(), fileNames.get(i));
+      assertFalse(mStore.get(fileIds.get(i)).isPresent());
+      assertFalse(mStore.getChild(mRoot.getId(), fileNames.get(i)).isPresent());
+    }
+  }
+
   @Test
   public void manyOperations() {
     writeInode(mRoot);

microbench/src/main/java/alluxio/inode/InodeBenchBase.java

@@ -40,6 +40,7 @@
 import alluxio.master.file.meta.options.MountInfo;
 import alluxio.master.journal.NoopJournalContext;
 import alluxio.master.metastore.InodeStore;
+import alluxio.master.metastore.caching.BasicInodeCache;
 import alluxio.master.metastore.caching.CachingInodeStore;
 import alluxio.master.metastore.heap.HeapInodeStore;
 import alluxio.master.metastore.rocks.RocksInodeStore;
@@ -60,6 +61,7 @@ class InodeBenchBase {
   public static final String HEAP = "heap";
   public static final String ROCKS = "rocks";
   public static final String ROCKSCACHE = "rocksCache";
+  public static final String BASICCACHE = "basicCache";
   private static final CreateDirectoryContext DIRECTORY_CONTEXT = CreateDirectoryContext
       .mergeFrom(CreateDirectoryPOptions.newBuilder().setMode(TEST_DIR_MODE.toProto()))
       .setOwner(TEST_OWNER).setGroup(TEST_GROUP);
@@ -115,6 +117,10 @@ static InodeStore getInodeStore(
         dir =
             AlluxioTestDirectory.createTemporaryDirectory("inode-store-bench").getAbsolutePath();
         return new CachingInodeStore(new RocksInodeStore(dir), lockManager);
+      case BASICCACHE:
+        dir =
+            AlluxioTestDirectory.createTemporaryDirectory("inode-store-bench").getAbsolutePath();
+        return new BasicInodeCache(dir);
       default:
         throw new IllegalStateException("Invalid type: " + inodeStoreType);
     }