fs/memory-provider: passed all apitests

Author: XiaochenCui

src/backend/exports.js

@@ -20,6 +20,7 @@ const CoreModule = require("./src/CoreModule.js");
 const { Kernel } = require("./src/Kernel.js");
 const DatabaseModule = require("./src/DatabaseModule.js");
 const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
+const MemoryStorageModule = require("./src/MemoryStorageModule.js");
 const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js");
 const { testlaunch } = require("./src/index.js");
 const BaseService = require("./src/services/BaseService.js");
@@ -71,6 +72,7 @@ module.exports = {
     WebModule,
     DatabaseModule,
     LocalDiskStorageModule,
+    MemoryStorageModule,
     SelfHostedModule,
     TestDriversModule,
     PuterAIModule,

src/backend/src/MemoryStorageModule.js

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class MemoryStorageModule {
+    async install (context) {
+        const services = context.get('services');
+        const MemoryStorageService = require("./services/MemoryStorageService");
+        services.registerService('memory-storage', MemoryStorageService);
+    }
+}
+
+module.exports = MemoryStorageModule;

src/backend/src/api/APIError.js

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const { URLSearchParams } = require("node:url");
+const config = require("../config");
 const { quot } = require('@heyputer/putility').libs.string;
 
 /**
@@ -518,14 +519,19 @@ module.exports = class APIError {
      * is set to null. The first argument is used as the status code.
      *
      * @static
-     * @param {number} status
-     * @param {string|Error} message_or_source one of the following:
+     * @param {number|string} status
+     * @param {object} source
+     * @param {string|Error|object} fields one of the following:
      * - a string to use as the error message
      * - an Error object to use as the source of the error
      * - an object with a message property to use as the error message
      * @returns
      */
     static create (status, source, fields = {}) {
+        if ( config.env === 'dev' ) {
+            console.trace('APIError.create', status, source, fields);
+        }
+
         // Just the error code
         if ( typeof status === 'string' ) {
             const code = this.codes[status];

src/backend/src/filesystem/FSNodeContext.js

@@ -288,7 +288,7 @@ module.exports = class FSNodeContext {
             controls,
         });
 
-        if ( entry === null ) {
+        if ( ! entry ) {
             this.found = false;
             this.entry = false;
         } else {

src/backend/src/filesystem/hl_operations/hl_copy.js

@@ -159,8 +159,8 @@ class HLCopy extends HLFilesystemOperation {
             throw APIError.create('source_and_dest_are_the_same');
         }
 
-        if ( await is_ancestor_of(source.mysql_id, parent.mysql_id) ) {
-            throw APIError('cannot_copy_item_into_itself');
+        if ( await is_ancestor_of(source.uid, parent.uid) ) {
+            throw APIError.create('cannot_copy_item_into_itself');
         }
 
         let overwritten;

src/backend/src/filesystem/hl_operations/hl_mkdir.js

@@ -287,7 +287,7 @@ class HLMkdir extends HLFilesystemOperation {
         // "top_parent" is the immediate parent of the target directory
         // (e.g: /home/foo/bar -> /home/foo)
         const top_parent = values.create_missing_parents
-            ? await this._create_top_parent({ top_parent: parent_node })
+            ? await this._create_dir(parent_node)
             : await this._get_existing_top_parent({ top_parent: parent_node })
             ;
 
@@ -331,12 +331,14 @@ class HLMkdir extends HLFilesystemOperation {
                 });
             }
             else if ( dedupe_name ) {
-                const fsEntryFetcher = context.get('services').get('fsEntryFetcher');
+                const fs = context.get('services').get('filesystem');
+                const parent_selector = parent_node.selector;
                 for ( let i=1 ;; i++ ) {
                     let try_new_name = `${target_basename} (${i})`;
-                    const exists = await fsEntryFetcher.nameExistsUnderParent(
-                        existing.entry.parent_uid, try_new_name
-                    );
+                    const selector = new NodeChildSelector(parent_selector, try_new_name);
+                    const exists = await parent_node.provider.quick_check({
+                        selector,
+                    });
                     if ( ! exists ) {
                         target_basename = try_new_name;
                         break;
@@ -468,22 +470,30 @@ class HLMkdir extends HLFilesystemOperation {
         return node;
     }
 
-    async _create_top_parent ({ top_parent }) {
-        if ( await top_parent.exists() ) {
-            if ( ! top_parent.entry.is_dir ) {
+    /**
+     * Creates a directory and all its ancestors.
+     *
+     * @param {FSNodeContext} dir - The directory to create.
+     * @returns {Promise<FSNodeContext>} The created directory.
+     */
+    async _create_dir (dir) {
+        console.log('CREATING DIR', dir.selector.describe());
+
+        if ( await dir.exists() ) {
+            if ( ! dir.entry.is_dir ) {
                 throw APIError.create('dest_is_not_a_directory');
             }
-            return top_parent;
+            return dir;
         }
 
         const maybe_path_selector =
-            top_parent.get_selector_of_type(NodePathSelector);
+            dir.get_selector_of_type(NodePathSelector);
 
         if ( ! maybe_path_selector ) {
             throw APIError.create('dest_does_not_exist');
         }
 
-        const path = maybe_path_selector.value;
+        let path = maybe_path_selector.value;
 
         const fs = this.context.get('services').get('filesystem');
 

src/backend/src/filesystem/ll_operations/ll_read.js

@@ -18,6 +18,7 @@
  */
 const APIError = require("../../api/APIError");
 const { Sequence } = require("../../codex/Sequence");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
 
 const { DB_WRITE } = require("../../services/database/consts");
 const { buffer_to_stream } = require("../../util/streamutil");
@@ -115,10 +116,13 @@ class LLRead extends LLFilesystemOperation {
             },
             async function create_S3_read_stream (a) {
                 const context = a.iget('context');
-                const storage = context.get('storage');
 
                 const { fsNode, version_id, offset, length, has_range } = a.values();
 
+                const svc_mountpoint = context.get('services').get('mountpoint');
+                const provider = await svc_mountpoint.get_provider(fsNode.selector);
+                const storage = svc_mountpoint.get_storage(provider.constructor);
+
                 // Empty object here is in the case of local fiesystem,
                 // where s3:location will return null.
                 // TODO: storage interface shouldn't have S3-specific properties.
@@ -133,6 +137,8 @@ class LLRead extends LLFilesystemOperation {
                     ...(has_range ? {
                         range: `bytes=${offset}-${offset+length-1}`
                     } : {}),
+
+                    memory_file: fsNode.entry,
                 }));
 
                 a.set('stream', stream);
@@ -144,8 +150,11 @@ class LLRead extends LLFilesystemOperation {
                 const { fsNode, stream, has_range } = a.values();
 
                 if ( ! has_range ) {
-                    const res = await svc_fileCache.maybe_store(fsNode, stream);
-                    if ( res.stream ) a.set('stream', res.stream);
+                    // only cache for non-memoryfs providers
+                    if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
+                        const res = await svc_fileCache.maybe_store(fsNode, stream);
+                        if ( res.stream ) a.set('stream', res.stream);
+                    }
                 }
             },
             async function return_stream (a) {

src/backend/src/filesystem/ll_operations/ll_rmdir.js

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const APIError = require("../../api/APIError");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
 const { ParallelTasks } = require("../../util/otelutil");
 const FSNodeContext = require("../FSNodeContext");
 const { NodeUIDSelector } = require("../node/selectors");
@@ -102,14 +103,27 @@ class LLRmDir extends LLFilesystemOperation {
         }
 
         await tasks.awaitAll();
-        if ( ! descendants_only ) {
-            await target.provider.rmdir({
+
+        // TODO (xiaochen): consolidate these two branches
+        if ( target.provider instanceof MemoryFSProvider ) {
+            await target.provider.rmdir( {
                 context,
                 node: target,
                 options: {
-                    ignore_not_empty: true,
+                    recursive,
+                    descendants_only,
                 },
-            });
+            } );
+        } else {
+            if ( ! descendants_only ) {
+                await target.provider.rmdir( {
+                    context,
+                    node: target,
+                    options: {
+                        ignore_not_empty: true,
+                    },
+                } );
+            }
         }
     }
 }

src/backend/src/filesystem/node/selectors.js

@@ -19,8 +19,14 @@
 const _path = require('path');
 const { PuterPath } = require('../lib/PuterPath');
 
-class NodePathSelector {
+class NodeSelector {
+    path = null;
+    uid = null;
+}
+
+class NodePathSelector extends NodeSelector {
     constructor (path) {
+        super();
         this.value = path;
     }
 
@@ -34,8 +40,9 @@ class NodePathSelector {
     }
 }
 
-class NodeUIDSelector {
+class NodeUIDSelector extends NodeSelector {
     constructor (uid) {
+        super();
         this.value = uid;
     }
 
@@ -58,8 +65,9 @@ class NodeUIDSelector {
     }
 }
 
-class NodeInternalIDSelector {
+class NodeInternalIDSelector extends NodeSelector {
     constructor (service, id, debugInfo) {
+        super();
         this.service = service;
         this.id = id;
         this.debugInfo = debugInfo;
@@ -81,23 +89,28 @@ class NodeInternalIDSelector {
     }
 }
 
-class NodeChildSelector {
+class NodeChildSelector extends NodeSelector {
     constructor (parent, name) {
+        super();
         this.parent = parent;
         this.name = name;
     }
 
     setPropertiesKnownBySelector (node) {
         node.name = this.name;
-        // no properties known
+
+        try_infer_attributes(this);
+        if ( this.path ) {
+            node.path = this.path;
+        }
     }
 
     describe () {
         return this.parent.describe() + '/' + this.name;
     }
 }
 
-class RootNodeSelector {
+class RootNodeSelector extends NodeSelector {
     static entry = {
         is_dir: true,
         is_root: true,
@@ -110,6 +123,7 @@ class RootNodeSelector {
         node.uid = PuterPath.NULL_UUID;
     }
     constructor () {
+        super();
         this.entry = this.constructor.entry;
     }
 
@@ -118,8 +132,9 @@ class RootNodeSelector {
     }
 }
 
-class NodeRawEntrySelector {
+class NodeRawEntrySelector extends NodeSelector {
     constructor (entry) {
+        super();
         // Fix entries from get_descendants
         if ( ! entry.uuid && entry.uid ) {
             entry.uuid = entry.uid;
@@ -145,6 +160,30 @@ class NodeRawEntrySelector {
     }
 }
 
+/**
+ * Try to infer following attributes for a selector:
+ * - path
+ * - uid
+ *
+ * @param {NodeSelector} selector
+ */
+function try_infer_attributes (selector) {
+    if ( selector instanceof NodePathSelector ) {
+        selector.path = selector.value;
+    } else if ( selector instanceof NodeUIDSelector ) {
+        selector.uid = selector.value;
+    } else if ( selector instanceof NodeChildSelector ) {
+        try_infer_attributes(selector.parent);
+        if ( selector.parent.path ) {
+            selector.path = _path.join(selector.parent.path, selector.name);
+        }
+    } else if ( selector instanceof RootNodeSelector ) {
+        selector.path = '/';
+    } else {
+        // give up
+    }
+}
+
 const relativeSelector = (parent, path) => {
     if ( path === '.' ) return parent;
     if ( path.startsWith('..') ) {
@@ -162,11 +201,13 @@ const relativeSelector = (parent, path) => {
 }
 
 module.exports = {
+    NodeSelector,
     NodePathSelector,
     NodeUIDSelector,
     NodeInternalIDSelector,
     NodeChildSelector,
     RootNodeSelector,
     NodeRawEntrySelector,
     relativeSelector,
+    try_infer_attributes,
 };