fs/memory-provider: write, read

Author: XiaochenCui

src/backend/exports.js

@@ -20,6 +20,7 @@ const CoreModule = require("./src/CoreModule.js");
 const { Kernel } = require("./src/Kernel.js");
 const DatabaseModule = require("./src/DatabaseModule.js");
 const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js");
+const MemoryStorageModule = require("./src/MemoryStorageModule.js");
 const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js");
 const { testlaunch } = require("./src/index.js");
 const BaseService = require("./src/services/BaseService.js");
@@ -71,6 +72,7 @@ module.exports = {
     WebModule,
     DatabaseModule,
     LocalDiskStorageModule,
+    MemoryStorageModule,
     SelfHostedModule,
     TestDriversModule,
     PuterAIModule,

src/backend/src/MemoryStorageModule.js

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+class MemoryStorageModule {
+    async install (context) {
+        const services = context.get('services');
+        const MemoryStorageService = require("./services/MemoryStorageService");
+        services.registerService('memory-storage', MemoryStorageService);
+    }
+}
+
+module.exports = MemoryStorageModule;

src/backend/src/filesystem/hl_operations/hl_read.js

@@ -34,6 +34,10 @@ class HLRead extends HLFilesystemOperation {
             version_id,
         } = this.values;
 
+        if ( fsNode.path.includes('tmp') ) {
+            console.log('tmp');
+        }
+
         if ( ! await fsNode.exists() ) {
             throw APIError.create('subject_does_not_exist');
         }

src/backend/src/filesystem/ll_operations/ll_read.js

@@ -18,6 +18,7 @@
  */
 const APIError = require("../../api/APIError");
 const { Sequence } = require("../../codex/Sequence");
+const { MemoryFSProvider } = require("../../modules/puterfs/customfs/MemoryFSProvider");
 
 const { DB_WRITE } = require("../../services/database/consts");
 const { buffer_to_stream } = require("../../util/streamutil");
@@ -70,6 +71,9 @@ class LLRead extends LLFilesystemOperation {
             async function calculate_has_range (a) {
                 const { offset, length } = a.values();
                 const fsNode = a.get('fsNode');
+                if ( fsNode.entry.id === 123 ) {
+                    console.log('has_range', offset, length);
+                }
                 const has_range = (
                     offset !== undefined &&
                     offset !== 0
@@ -115,10 +119,13 @@ class LLRead extends LLFilesystemOperation {
             },
             async function create_S3_read_stream (a) {
                 const context = a.iget('context');
-                const storage = context.get('storage');
 
                 const { fsNode, version_id, offset, length, has_range } = a.values();
 
+                const svc_mountpoint = context.get('services').get('mountpoint');
+                const provider = await svc_mountpoint.get_provider(fsNode.selector);
+                const storage = svc_mountpoint.get_storage(provider.constructor);
+
                 // Empty object here is in the case of local fiesystem,
                 // where s3:location will return null.
                 // TODO: storage interface shouldn't have S3-specific properties.
@@ -133,6 +140,8 @@ class LLRead extends LLFilesystemOperation {
                     ...(has_range ? {
                         range: `bytes=${offset}-${offset+length-1}`
                     } : {}),
+
+                    memory_file: fsNode.entry,
                 }));
 
                 a.set('stream', stream);
@@ -144,8 +153,10 @@ class LLRead extends LLFilesystemOperation {
                 const { fsNode, stream, has_range } = a.values();
 
                 if ( ! has_range ) {
-                    const res = await svc_fileCache.maybe_store(fsNode, stream);
-                    if ( res.stream ) a.set('stream', res.stream);
+                    if ( ! (fsNode.provider instanceof MemoryFSProvider) ) {
+                        const res = await svc_fileCache.maybe_store(fsNode, stream);
+                        if ( res.stream ) a.set('stream', res.stream);
+                    }
                 }
             },
             async function return_stream (a) {

src/backend/src/modules/puterfs/MountpointService.js

@@ -57,8 +57,9 @@ class MountpointService extends BaseService {
     * @returns {Promise<void>}
     */
     async _init () {
-        // Temporary solution - we'll develop this incrementally
-        this.storage_ = null;
+        // key: provider class (e.g: PuterFSProvider, MemoryFSProvider)
+        // value: storage instance
+        this.storage_ = {};
     }
 
     async ['__on_boot.consolidation'] () {
@@ -145,15 +146,16 @@ class MountpointService extends BaseService {
     }
 
     // Temporary solution - we'll develop this incrementally
-    set_storage (storage) {
-        this.storage_ = storage;
+    set_storage (provider, storage) {
+        this.storage_[provider] = storage;
     }
+
     /**
     * Gets the current storage backend instance
     * @returns {Object} The storage backend instance
     */
-    get_storage () {
-        return this.storage_;
+    get_storage (provider) {
+        return this.storage_[provider];
     }
 }
 

src/backend/src/modules/puterfs/customfs/MemoryFSProvider.js

@@ -22,7 +22,8 @@ const _path = require('path');
 const { Context } = require("../../../util/context");
 const { v4: uuidv4 } = require('uuid');
 const config = require("../../../config");
-const { try_infer_attributes, NodeChildSelector, NodePathSelector, NodeSelector } = require("../../../filesystem/node/selectors");
+const { try_infer_attributes, NodeChildSelector, NodePathSelector, NodeUIDSelector, NodeSelector } = require("../../../filesystem/node/selectors");
+const fsCapabilities = require("../../../filesystem/definitions/capabilities");
 
 const path = require('path');
 const APIError = require("../../../api/APIError");
@@ -62,11 +63,10 @@ class MemoryFile {
         this.is_shortcut = 0;
         this.is_symlink = 0;
         this.symlink_path = null;
-        this.created = 123;
-        this.accessed = 123;
-        this.modified = 123;
-
-        this.own
+        this.created = Math.floor(Date.now() / 1000);
+        this.accessed = Math.floor(Date.now() / 1000);
+        this.modified = Math.floor(Date.now() / 1000);
+        this.size = is_dir ? 0 : (content ? content.length : 0);
     }
 }
 
@@ -94,6 +94,20 @@ class MemoryFSProvider {
         this.entriesByUUID.set(root.uuid, root);
     }
 
+    /**
+     * Get the capabilities of this filesystem provider.
+     * 
+     * @returns {Set} - Set of capabilities supported by this provider.
+     */
+    get_capabilities() {
+        return new Set([
+            fsCapabilities.READDIR_UUID_MODE,
+            fsCapabilities.UUID,
+            fsCapabilities.READ,
+            fsCapabilities.WRITE,
+        ]);
+    }
+
     /**
      * Normalize the path to be relative to the mountpoint. Returns `/` if the path is empty/undefined.
      * 
@@ -119,7 +133,7 @@ class MemoryFSProvider {
     /**
      * Check the integrity of the whole memory filesystem and the input. Throws error if any violation is found.
      * 
-     * @param {MemoryFile} entry - The entry to check.
+     * @param {MemoryFile|FSNodeContext} entry - The entry to check.
      * @returns {Promise<void>}
      */
     _integrity_check (entry) {
@@ -130,7 +144,8 @@ class MemoryFSProvider {
 
         if ( entry ) {
             // check all directories along the path are valid
-            const inner_path = this._inner_path(entry.path);
+            const path_to_check = 'entry' in entry ? entry.entry?.path : entry.path;
+            const inner_path = this._inner_path(path_to_check);
             const path_components = inner_path.split('/');
             for ( let i = 2; i < path_components.length; i++ ) {
                 const path_component = path_components.slice(0, i).join('/');
@@ -211,6 +226,33 @@ class MemoryFSProvider {
         return entry;
     }
 
+    /**
+     * Read directory contents.
+     * 
+     * @param {Object} param
+     * @param {Context} param.context - The context of the operation.
+     * @param {FSNodeContext} param.node - The directory node to read.
+     * @returns {Promise<string[]>} - Array of child UUIDs.
+     */
+    async readdir({ context, node }) {
+        const inner_path = this._inner_path(node.path);
+        const child_uuids = [];
+
+        // Find all entries that are direct children of this directory
+        for (const [path, uuid] of this.entriesByPath) {
+            if (path === inner_path) {
+                continue; // Skip the directory itself
+            }
+            
+            const dirname = _path.dirname(path);
+            if (dirname === inner_path) {
+                child_uuids.push(uuid);
+            }
+        }
+
+        return child_uuids;
+    }
+
     /**
      * Create a new directory.
      * 
@@ -301,7 +343,7 @@ class MemoryFSProvider {
         const new_inner_path = this._inner_path(new_full_path);
         const entry = new MemoryFile({
             full_path: new_full_path,
-            is_dir: false,
+            is_dir: node.entry.is_dir,
             content: node.entry.content,
         });
         entry.uuid = node.entry.uuid;
@@ -318,6 +360,49 @@ class MemoryFSProvider {
         return entry;
     }
 
+    /**
+     * Copy a tree of files and directories.
+     * 
+     * @param {Object} param
+     * @param {Context} param.context
+     * @param {FSNodeContext} param.source - The source node to copy.
+     * @param {FSNodeContext} param.parent - The parent directory for the copy.
+     * @param {string} param.target_name - The name for the copied item.
+     * @returns {Promise<FSNodeContext>} - The copied node.
+     */
+    async copy_tree({ context, source, parent, target_name }) {
+        const fs = context.get('services').get('filesystem');
+        
+        if (source.entry.is_dir) {
+            // Create the directory
+            const new_dir = await this.mkdir({ context, parent, name: target_name });
+            
+            // Copy all children
+            const children = await this.readdir({ context, node: source });
+            for (const child_uuid of children) {
+                const child_node = await fs.node(new NodeUIDSelector(child_uuid));
+                const child_name = child_node.entry.name;
+                await this.copy_tree({ 
+                    context, 
+                    source: child_node, 
+                    parent: new_dir, 
+                    target_name: child_name 
+                });
+            }
+            
+            return new_dir;
+        } else {
+            // Copy the file
+            const new_file = await this.write_new({ 
+                context, 
+                parent, 
+                name: target_name, 
+                file: { stream: { read: () => source.entry.content } } 
+            });
+            return new_file;
+        }
+    }
+
     /**
      * Write a new file to the filesystem. Throws an error if the destination
      * already exists.
@@ -336,15 +421,14 @@ class MemoryFSProvider {
         const entry = new MemoryFile({
             full_path: full_path,
             is_dir: false,
-            content: file.stream.readableBuffer,
+            content: file.stream.read(),
         });
         this.entriesByPath.set(inner_path, entry.uuid);
         this.entriesByUUID.set(entry.uuid, entry);
 
         const fs = context.get('services').get('filesystem');
         const node = await fs.node(entry.path);
-
-        this._integrity_check(node);
+        await node.fetchEntry();
 
         return node;
     }
@@ -371,12 +455,19 @@ class MemoryFSProvider {
                 throw new Error(`Cannot overwrite a directory`);
             }
 
-            original_entry.content = file.stream.readableBuffer;
+            original_entry.content = file.stream.read();
+            original_entry.modified = Math.floor(Date.now() / 1000);
+            original_entry.size = original_entry.content ? original_entry.content.length : 0;
             this.entriesByUUID.set(node.uid, original_entry);
         }
 
         this._integrity_check(node);
 
+        // return node;
+        const fs = context.get('services').get('filesystem');
+        node = await fs.node(original_entry.path);
+        await node.fetchEntry();
+
         return node;
     }
 }

src/backend/src/modules/puterfs/lib/PuterFSProvider.js

@@ -761,7 +761,7 @@ class PuterFSProvider extends putility.AdvancedBase {
         const svc_event = svc.get('event');
 
         const svc_mountpoint = svc.get('mountpoint');
-        const storage = svc_mountpoint.get_storage();
+        const storage = svc_mountpoint.get_storage(this.constructor);
 
         bucket        ??= config.s3_bucket;
         bucket_region ??= config.s3_region ?? config.region;