dev: memory filesystem

* fs/memory-provider: passed all apitests

* test: write benchmark

* fs: add benchmark for stat-intensive scenario

* apitest: update duration

* fs: remove "NodeSelector", add checks to memoryfs, passed simple test

* test: update apitest

* debug: remove a debug stmt

Author: XiaochenCui

src/backend/exports.js

@@ -20,6 +20,7 @@ const CoreModule = require("./src/CoreModule.js");
 const { Kernel } = require("./src/Kernel.js");
 const DatabaseModule = require("./src/DatabaseModule.js");
 const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
+const MemoryStorageModule = require("./src/MemoryStorageModule.js");
 const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js");
 const { testlaunch } = require("./src/index.js");
 const BaseService = require("./src/services/BaseService.js");
@@ -73,6 +74,7 @@ module.exports = {
     WebModule,
     DatabaseModule,
     LocalDiskStorageModule,
+    MemoryStorageModule,
     SelfHostedModule,
     TestDriversModule,
     PuterAIModule,

src/backend/src/MemoryStorageModule.js

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class MemoryStorageModule {
+    async install (context) {
+        const services = context.get('services');
+        const MemoryStorageService = require("./services/MemoryStorageService");
+        services.registerService('memory-storage', MemoryStorageService);
+    }
+}
+
+module.exports = MemoryStorageModule;

src/backend/src/api/APIError.js

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const { URLSearchParams } = require("node:url");
+const config = require("../config");
 const { quot } = require('@heyputer/putility').libs.string;
 
 /**
@@ -518,8 +519,9 @@ module.exports = class APIError {
      * is set to null. The first argument is used as the status code.
      *
      * @static
-     * @param {number} status
-     * @param {string|Error} message_or_source one of the following:
+     * @param {number|string} status
+     * @param {object} source
+     * @param {string|Error|object} fields one of the following:
      * - a string to use as the error message
      * - an Error object to use as the source of the error
      * - an object with a message property to use as the error message

src/backend/src/filesystem/FSNodeContext.js

@@ -288,7 +288,7 @@ module.exports = class FSNodeContext {
             controls,
         });
 
-        if ( entry === null ) {
+        if ( ! entry ) {
             this.found = false;
             this.entry = false;
         } else {

src/backend/src/filesystem/hl_operations/hl_copy.js

@@ -159,8 +159,8 @@ class HLCopy extends HLFilesystemOperation {
             throw APIError.create('source_and_dest_are_the_same');
         }
 
-        if ( await is_ancestor_of(source.mysql_id, parent.mysql_id) ) {
-            throw APIError('cannot_copy_item_into_itself');
+        if ( await is_ancestor_of(source.uid, parent.uid) ) {
+            throw APIError.create('cannot_copy_item_into_itself');
         }
 
         let overwritten;

src/backend/src/filesystem/hl_operations/hl_mkdir.js

@@ -287,7 +287,7 @@ class HLMkdir extends HLFilesystemOperation {
         // "top_parent" is the immediate parent of the target directory
         // (e.g: /home/foo/bar -> /home/foo)
         const top_parent = values.create_missing_parents
-            ? await this._create_top_parent({ top_parent: parent_node })
+            ? await this._create_dir(parent_node)
             : await this._get_existing_top_parent({ top_parent: parent_node })
             ;
 
@@ -331,12 +331,14 @@ class HLMkdir extends HLFilesystemOperation {
                 });
             }
             else if ( dedupe_name ) {
-                const fsEntryFetcher = context.get('services').get('fsEntryFetcher');
+                const fs = context.get('services').get('filesystem');
+                const parent_selector = parent_node.selector;
                 for ( let i=1 ;; i++ ) {
                     let try_new_name = `${target_basename} (${i})`;
-                    const exists = await fsEntryFetcher.nameExistsUnderParent(
-                        existing.entry.parent_uid, try_new_name
-                    );
+                    const selector = new NodeChildSelector(parent_selector, try_new_name);
+                    const exists = await parent_node.provider.quick_check({
+                        selector,
+                    });
                     if ( ! exists ) {
                         target_basename = try_new_name;
                         break;
@@ -468,16 +470,24 @@ class HLMkdir extends HLFilesystemOperation {
         return node;
     }
 
-    async _create_top_parent ({ top_parent }) {
-        if ( await top_parent.exists() ) {
-            if ( ! top_parent.entry.is_dir ) {
+    /**
+     * Creates a directory and all its ancestors.
+     *
+     * @param {FSNodeContext} dir - The directory to create.
+     * @returns {Promise<FSNodeContext>} The created directory.
+     */
+    async _create_dir (dir) {
+        console.log('CREATING DIR', dir.selector.describe());
+
+        if ( await dir.exists() ) {
+            if ( ! dir.entry.is_dir ) {
                 throw APIError.create('dest_is_not_a_directory');
             }
-            return top_parent;
+            return dir;
         }
 
         const maybe_path_selector =
-            top_parent.get_selector_of_type(NodePathSelector);
+            dir.get_selector_of_type(NodePathSelector);
 
         if ( ! maybe_path_selector ) {
             throw APIError.create('dest_does_not_exist');

src/backend/src/filesystem/ll_operations/ll_read.js

@@ -18,6 +18,7 @@
  */
 const APIError = require("../../api/APIError");
 const { Sequence } = require("../../codex/Sequence");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
 
 const { DB_WRITE } = require("../../services/database/consts");
 const { buffer_to_stream } = require("../../util/streamutil");
@@ -115,10 +116,13 @@ class LLRead extends LLFilesystemOperation {
             },
             async function create_S3_read_stream (a) {
                 const context = a.iget('context');
-                const storage = context.get('storage');
 
                 const { fsNode, version_id, offset, length, has_range, range } = a.values();
 
+                const svc_mountpoint = context.get('services').get('mountpoint');
+                const provider = await svc_mountpoint.get_provider(fsNode.selector);
+                const storage = svc_mountpoint.get_storage(provider.constructor);
+
                 // Empty object here is in the case of local fiesystem,
                 // where s3:location will return null.
                 // TODO: storage interface shouldn't have S3-specific properties.
@@ -130,6 +134,7 @@ class LLRead extends LLFilesystemOperation {
                     bucket_region: location.bucket_region,
                     version_id,
                     key: location.key,
+                    memory_file: fsNode.entry,
                     ...(range? {range} : (has_range ? {
                         range: `bytes=${offset}-${offset+length-1}`
                     } : {})),
@@ -144,8 +149,11 @@ class LLRead extends LLFilesystemOperation {
                 const { fsNode, stream, has_range, range} = a.values();
 
                 if ( ! has_range ) {
-                    const res = await svc_fileCache.maybe_store(fsNode, stream);
-                    if ( res.stream ) a.set('stream', res.stream);
+                    // only cache for non-memoryfs providers
+                    if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
+                        const res = await svc_fileCache.maybe_store(fsNode, stream);
+                        if ( res.stream ) a.set('stream', res.stream);
+                    }
                 }
             },
             async function return_stream (a) {

src/backend/src/filesystem/ll_operations/ll_rmdir.js

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const APIError = require("../../api/APIError");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
 const { ParallelTasks } = require("../../util/otelutil");
 const FSNodeContext = require("../FSNodeContext");
 const { NodeUIDSelector } = require("../node/selectors");
@@ -102,14 +103,27 @@ class LLRmDir extends LLFilesystemOperation {
         }
 
         await tasks.awaitAll();
-        if ( ! descendants_only ) {
-            await target.provider.rmdir({
+
+        // TODO (xiaochen): consolidate these two branches
+        if ( target.provider instanceof MemoryFSProvider ) {
+            await target.provider.rmdir( {
                 context,
                 node: target,
                 options: {
-                    ignore_not_empty: true,
+                    recursive,
+                    descendants_only,
                 },
-            });
+            } );
+        } else {
+            if ( ! descendants_only ) {
+                await target.provider.rmdir( {
+                    context,
+                    node: target,
+                    options: {
+                        ignore_not_empty: true,
+                    },
+                } );
+            }
         }
     }
 }

src/backend/src/filesystem/node/selectors.js

@@ -89,7 +89,11 @@ class NodeChildSelector {
 
     setPropertiesKnownBySelector (node) {
         node.name = this.name;
-        // no properties known
+
+        try_infer_attributes(this);
+        if ( this.path ) {
+            node.path = this.path;
+        }
     }
 
     describe () {
@@ -145,6 +149,30 @@ class NodeRawEntrySelector {
     }
 }
 
+/**
+ * Try to infer following attributes for a selector:
+ * - path
+ * - uid
+ *
+ * @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} selector
+ */
+function try_infer_attributes (selector) {
+    if ( selector instanceof NodePathSelector ) {
+        selector.path = selector.value;
+    } else if ( selector instanceof NodeUIDSelector ) {
+        selector.uid = selector.value;
+    } else if ( selector instanceof NodeChildSelector ) {
+        try_infer_attributes(selector.parent);
+        if ( selector.parent.path ) {
+            selector.path = _path.join(selector.parent.path, selector.name);
+        }
+    } else if ( selector instanceof RootNodeSelector ) {
+        selector.path = '/';
+    } else {
+        // give up
+    }
+}
+
 const relativeSelector = (parent, path) => {
     if ( path === '.' ) return parent;
     if ( path.startsWith('..') ) {
@@ -169,4 +197,5 @@ module.exports = {
     RootNodeSelector,
     NodeRawEntrySelector,
     relativeSelector,
+    try_infer_attributes,
 };

src/backend/src/helpers.js

@@ -966,7 +966,38 @@ const body_parser_error_handler = (err, req, res, next) => {
     next();
 }
 
+/**
+ * Given a uid, returns a file node.
+ * 
+ * TODO (xiaochen): It only works for MemoryFSProvider currently.
+ * 
+ * @param {string} uid - The uid of the file to get.
+ * @returns {Promise<MemoryFile|null>} The file node, or null if the file does not exist.
+ */
+async function get_entry(uid) {
+    const svc_mountpoint = Context.get('services').get('mountpoint');
+    const uid_selector = new NodeUIDSelector(uid);
+    const provider = await svc_mountpoint.get_provider(uid_selector);
+
+    // NB: We cannot import MemoryFSProvider here because it will cause a circular dependency.
+    if ( provider.constructor.name !== 'MemoryFSProvider' ) {
+        return null;
+    }
+
+    return provider.stat({
+        selector: uid_selector,
+    });
+}
+
 async function is_ancestor_of(ancestor_uid, descendant_uid){
+    const ancestor = await get_entry(ancestor_uid);
+    const descendant = await get_entry(descendant_uid);
+
+    if ( ancestor && descendant ) {
+        return descendant.path.startsWith(ancestor.path);
+    }
+
+
     /** @type BaseDatabaseAccessService */
     const db = services.get('database').get(DB_READ, 'filesystem');