diff --git a/extensions/puterfs/PuterFSProvider.js b/extensions/puterfs/PuterFSProvider.js index 7488ff408c..e99d31697e 100644 --- a/extensions/puterfs/PuterFSProvider.js +++ b/extensions/puterfs/PuterFSProvider.js @@ -38,7 +38,6 @@ const svc_acl = extension.import('service:acl'); // TODO: these services ought to be part of this extension const svc_size = extension.import('service:sizeService'); -const svc_fsEntryFetcher = extension.import('service:fsEntryFetcher'); const svc_resource = extension.import('service:resourceService'); // Not sure where these really belong yet @@ -98,8 +97,9 @@ const { } = extension.import('fs').util; export default class PuterFSProvider { - constructor ({ fsEntryController }) { + constructor ({ fsEntryController, storageController }) { this.fsEntryController = fsEntryController; + this.storageController = storageController; this.name = 'puterfs'; } @@ -219,25 +219,25 @@ export default class PuterFSProvider { }) { // shortcut: has full path if ( selector?.path ) { - const entry = await svc_fsEntryFetcher.findByPath(selector.path); + const entry = await this.fsEntryController.findByPath(selector.path); return Boolean(entry); } // shortcut: has uid if ( selector?.uid ) { - const entry = await svc_fsEntryFetcher.findByUID(selector.uid); + const entry = await this.fsEntryController.findByUID(selector.uid); return Boolean(entry); } // shortcut: parent uid + child name if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) { - return await svc_fsEntryFetcher.nameExistsUnderParent(selector.parent.uid, + return await this.fsEntryController.nameExistsUnderParent(selector.parent.uid, selector.name); } // shortcut: parent id + child name if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) { - return await svc_fsEntryFetcher.nameExistsUnderParentID(selector.parent.id, + return await this.fsEntryController.nameExistsUnderParentID(selector.parent.id, selector.name); } @@ -358,7 +358,7 @@ export default class PuterFSProvider { node, }) { // For Puter FS nodes, we assume we will obtain all properties from - // fsEntryService/fsEntryFetcher, except for 'thumbnail' unless it's + // fsEntryController, except for 'thumbnail' unless it's // explicitly requested. if ( options.tracer == null ) { @@ -409,7 +409,7 @@ export default class PuterFSProvider { entry = await this.fsEntryController.get(maybe_uid, options); controls.log.debug('got an entry from the future'); } else { - entry = await svc_fsEntryFetcher.find(selector, options); + entry = await this.fsEntryController.find(selector, options); } if ( ! entry ) { @@ -944,7 +944,7 @@ export default class PuterFSProvider { const state_upload = storage.create_upload(); try { - await state_upload.run({ + await this.storageController.upload({ uid: uuid, file, storage_meta: { bucket, bucket_region }, diff --git a/extensions/puterfs/fsentries/FSEntryController.js b/extensions/puterfs/fsentries/FSEntryController.js index d7a4a0422f..7828b39650 100644 --- a/extensions/puterfs/fsentries/FSEntryController.js +++ b/extensions/puterfs/fsentries/FSEntryController.js @@ -12,7 +12,11 @@ const { PuterPath } = extension.import('fs'); const { Context } = extension.import('core'); const { + RootNodeSelector, + NodeChildSelector, NodeUIDSelector, + NodePathSelector, + NodeInternalIDSelector, } = extension.import('core').fs.selectors; export default class { @@ -36,6 +40,37 @@ export default class { this.entryListeners_ = {}; this.mkPromiseForQueueSize_(); + + // this list of properties is for read operations + // (originally in FSEntryFetcher) + this.defaultProperties = [ + 'id', + 'associated_app_id', + 'uuid', + 'public_token', + 'bucket', + 'bucket_region', + 'file_request_token', + 'user_id', + 'parent_uid', + 'is_dir', + 'is_public', + 'is_shortcut', + 'is_symlink', + 'symlink_path', + 'shortcut_to', + 'sort_by', + 'sort_order', + 'immutable', + 'name', + 'metadata', + 'modified', + 'created', + 'accessed', + 'size', + 'layout', + 'path', + ]; } init () { @@ -67,6 +102,7 @@ export default class { }); } + // #region write operations async insert (entry) { const op = new Insert(entry); await this.enqueue_(op); @@ -84,7 +120,9 @@ export default class { await this.enqueue_(op); return op; } + // #endregion + // #region read operations async fast_get_descendants (uuid) { return (await db.read(` WITH RECURSIVE descendant_cte AS ( @@ -160,8 +198,7 @@ export default class { op.apply(answer); } if ( answer.is_diff ) { - const fsEntryFetcher = Context.get('services').get('fsEntryFetcher'); - const base_entry = await fsEntryFetcher.find(new NodeUIDSelector(uuid), + const base_entry = await this.find(new NodeUIDSelector(uuid), fetch_entry_options); answer.entry = { ...base_entry, ...answer.entry }; } @@ -197,6 +234,203 @@ export default class { return rows[0].total_size; } + /** + * Finds a filesystem entry using the provided selector. + * @param {Object} selector - The selector object specifying how to find the entry + * @param {Object} fetch_entry_options - Options for fetching the entry + * @returns {Promise} The filesystem entry or null if not found + */ + async find (selector, fetch_entry_options) { + if ( selector instanceof RootNodeSelector ) { + return selector.entry; + } + if ( selector instanceof NodePathSelector ) { + return await this.findByPath(selector.value, fetch_entry_options); + } + if ( selector instanceof NodeUIDSelector ) { + return await this.findByUID(selector.value, fetch_entry_options); + } + if ( selector instanceof NodeInternalIDSelector ) { + return await this.findByID(selector.id, fetch_entry_options); + } + if ( selector instanceof NodeChildSelector ) { + let id; + + if ( selector.parent instanceof RootNodeSelector ) { + id = await this.findNameInRoot(selector.name); + } else { + const parentEntry = await this.find(selector.parent); + if ( ! parentEntry ) return null; + id = await this.findNameInParent(parentEntry.uuid, selector.name); + } + + if ( id === undefined ) return null; + if ( typeof id !== 'number' ) { + throw new Error('unexpected type for id value', + typeof id, + id); + } + return this.find(new NodeInternalIDSelector('mysql', id)); + } + } + + /** + * Finds a filesystem entry by its UUID. + * @param {string} uuid - The UUID of the entry to find + * @param {Object} fetch_entry_options - Options including thumbnail flag + * @returns {Promise} The filesystem entry or undefined if not found + */ + async findByUID (uuid, fetch_entry_options = {}) { + const { thumbnail } = fetch_entry_options; + + let fsentry = await db.tryHardRead(`SELECT ${ + this.defaultProperties.join(', ') + }${thumbnail ? ', thumbnail' : '' + } FROM fsentries WHERE uuid = ? LIMIT 1`, + [uuid]); + + return fsentry[0]; + } + + /** + * Finds a filesystem entry by its internal database ID. + * @param {number} id - The internal ID of the entry to find + * @param {Object} fetch_entry_options - Options including thumbnail flag + * @returns {Promise} The filesystem entry or undefined if not found + */ + async findByID (id, fetch_entry_options = {}) { + const { thumbnail } = fetch_entry_options; + + let fsentry = await db.tryHardRead(`SELECT ${ + this.defaultProperties.join(', ') + }${thumbnail ? ', thumbnail' : '' + } FROM fsentries WHERE id = ? LIMIT 1`, + [id]); + + return fsentry[0]; + } + + /** + * Finds a filesystem entry by its full path. + * @param {string} path - The full path of the entry to find + * @param {Object} fetch_entry_options - Options including thumbnail flag and tracer + * @returns {Promise} The filesystem entry or false if not found + */ + async findByPath (path, fetch_entry_options = {}) { + const { thumbnail } = fetch_entry_options; + + if ( path === '/' ) { + return this.find(new RootNodeSelector()); + } + + const parts = path.split('/').filter(path => path !== ''); + if ( parts.length === 0 ) { + // TODO: invalid path; this should be an error + return false; + } + + // TODO: use a closure table for more efficient path resolving + let parent_uid = null; + let result; + + const resultColsSql = this.defaultProperties.join(', ') + + (thumbnail ? ', thumbnail' : ''); + + result = await db.read(`SELECT ${ resultColsSql + } FROM fsentries WHERE path=? LIMIT 1`, + [path]); + + // using knex instead + + if ( result[0] ) return result[0]; + + const loop = async () => { + for ( let i = 0 ; i < parts.length ; i++ ) { + const part = parts[i]; + const isLast = i == parts.length - 1; + const colsSql = isLast ? resultColsSql : 'uuid'; + if ( parent_uid === null ) { + result = await db.read(`SELECT ${ colsSql + } FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`, + [part]); + } else { + result = await db.read(`SELECT ${ colsSql + } FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`, + [parent_uid, part]); + } + + if ( ! result[0] ) return false; + parent_uid = result[0].uuid; + } + }; + + if ( fetch_entry_options.tracer ) { + const tracer = fetch_entry_options.tracer; + const options = fetch_entry_options.trace_options; + await tracer.startActiveSpan('fs:sql:findByPath', + ...(options ? [options] : []), + async span => { + await loop(); + span.end(); + }); + } else { + await loop(); + } + + return result[0]; + } + + /** + * Finds the ID of a child entry with the given name in the root directory. + * @param {string} name - The name of the child entry to find + * @returns {Promise} The ID of the child entry or undefined if not found + */ + async findNameInRoot (name) { + let child_id = await db.read('SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1', + [name]); + return child_id[0]?.id; + } + + /** + * Finds the ID of a child entry with the given name under a specific parent. + * @param {string} parent_uid - The UUID of the parent directory + * @param {string} name - The name of the child entry to find + * @returns {Promise} The ID of the child entry or undefined if not found + */ + async findNameInParent (parent_uid, name) { + let child_id = await db.read('SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1', + [parent_uid, name]); + return child_id[0]?.id; + } + + /** + * Checks if an entry with the given name exists under a specific parent. + * @param {string} parent_uid - The UUID of the parent directory + * @param {string} name - The name to check for + * @returns {Promise} True if the name exists under the parent, false otherwise + */ + async nameExistsUnderParent (parent_uid, name) { + let check_dupe = await db.read('SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1', + [parent_uid, name]); + return !!check_dupe[0]; + } + + /** + * Checks if an entry with the given name exists under a parent specified by ID. + * @param {number} parent_id - The internal ID of the parent directory + * @param {string} name - The name to check for + * @returns {Promise} True if the name exists under the parent, false otherwise + */ + async nameExistsUnderParentID (parent_id, name) { + const parent = await this.findByID(parent_id); + if ( ! parent ) { + return false; + } + return this.nameExistsUnderParent(parent.uuid, name); + } + // #endregion + + // #region queue logic async enqueue_ (op) { while ( this.currentState.queue.length > this.max_queue || @@ -296,4 +530,5 @@ export default class { this.mkPromiseForQueueSize_(); queueSizeResolve(); } + // #endregion } \ No newline at end of file diff --git a/extensions/puterfs/main.js b/extensions/puterfs/main.js index 2005547177..45e114266e 100644 --- a/extensions/puterfs/main.js +++ b/extensions/puterfs/main.js @@ -19,11 +19,49 @@ import FSEntryController from './fsentries/FSEntryController.js'; import PuterFSProvider from './PuterFSProvider.js'; +import LocalDiskStorageController from './storage/LocalDiskStorageController.js'; +import ProxyStorageController from './storage/ProxyStorageController.js'; + +const svc_event = extension.import('service:event'); const fsEntryController = new FSEntryController(); +const storageController = new ProxyStorageController(); -extension.on('init', () => { +extension.on('init', async () => { fsEntryController.init(); + + // Keep track of possible storage strategies for puterfs here + let defaultStorage = 'flat-files'; + const storageStrategies = { + 'flat-files': new LocalDiskStorageController(), + }; + + // Emit the "create storage strategies" event + const event = { + createStorageStrategy (name, implementation) { + storageStrategies[name] = implementation; + if ( implementation === undefined ) { + throw new Error('createStorageStrategy was called wrong'); + } + if ( implementation.forceDefault ) { + defaultStorage = name; + } + }, + }; + // Awaiting the event ensures all the storage strategies are registered + await svc_event.emit('puterfs.storage.create', event); + + let configuredStorage = defaultStorage; + if ( config.storage ) configuredStorage = config.storage; + + // Not we can select the configured strategy + const storageToUse = storageStrategies[configuredStorage]; + storageController.setDelegate(storageToUse); + + // The StorageController may need to await some asynchronous operations + // before it's ready to be used. + await storageController.init(); + }); extension.on('create.filesystem-types', event => { @@ -31,6 +69,7 @@ extension.on('create.filesystem-types', event => { mount ({ path }) { return new PuterFSProvider({ fsEntryController, + storageController, }); }, }); diff --git a/extensions/puterfs/storage/LocalDiskStorageController.js b/extensions/puterfs/storage/LocalDiskStorageController.js new file mode 100644 index 0000000000..9b1fa30705 --- /dev/null +++ b/extensions/puterfs/storage/LocalDiskStorageController.js @@ -0,0 +1,65 @@ +import putility from '@heyputer/putility'; +import fs from 'node:fs'; +import path_ from 'node:path'; + +const { + progress_stream, + size_limit_stream, +} = extension.import('core').util.streamutil; + +export default class LocalDiskStorageController { + constructor () { + this.path = path_.join(process.cwd(), '/storage'); + } + + async init () { + await fs.promises.mkdir(this.path, { recursive: true }); + } + + async upload ({ uid, file, storage_api }) { + const { progress_tracker } = storage_api; + + if ( file.buffer ) { + const path = this.#getPath(uid); + await fs.promises.writeFile(path, file.buffer); + + progress_tracker.set_total(file.buffer.length); + progress_tracker.set(file.buffer.length); + return; + } + + let stream = file.stream; + stream = progress_stream(stream, { + total: file.size, + progress_callback: evt => { + progress_tracker.set_total(file.size); + progress_tracker.set(evt.uploaded); + }, + }); + stream = size_limit_stream(stream, { + limit: file.size, + }); + + const writePromise = new putility.libs.promise.TeePromise(); + const path = this.#getPath(uid); + const write_stream = fs.createWriteStream(path); + + write_stream.on('error', () => writePromise.reject()); + write_stream.on('finish', () => writePromise.resolve()); + + stream.pipe(write_stream); + + // @ts-ignore (it's wrong about this) + await writePromise; + } + copy () { + } + delete () { + } + read () { + } + + #getPath (key) { + return path_.join(this.path, key); + } +} \ No newline at end of file diff --git a/extensions/puterfs/storage/ProxyStorageController.js b/extensions/puterfs/storage/ProxyStorageController.js new file mode 100644 index 0000000000..1627c18381 --- /dev/null +++ b/extensions/puterfs/storage/ProxyStorageController.js @@ -0,0 +1,24 @@ +export default class { + constructor (delegate) { + this.delegate = delegate ?? null; + } + setDelegate (delegate) { + this.delegate = delegate; + } + + init (...a) { + return this.delegate.init(...a); + } + upload (...a) { + return this.delegate.upload(...a); + } + copy (...a) { + return this.delegate.copy(...a); + } + delete (...a) { + return this.delegate.delete(...a); + } + read (...a) { + return this.delegate.read(...a); + } +} diff --git a/src/backend/src/filesystem/hl_operations/hl_write.js b/src/backend/src/filesystem/hl_operations/hl_write.js index cea9aad6ff..708b5c4412 100644 --- a/src/backend/src/filesystem/hl_operations/hl_write.js +++ b/src/backend/src/filesystem/hl_operations/hl_write.js @@ -16,23 +16,23 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const APIError = require("../../api/APIError"); -const FSNodeParam = require("../../api/filesystem/FSNodeParam"); -const FlagParam = require("../../api/filesystem/FlagParam"); -const StringParam = require("../../api/filesystem/StringParam"); -const UserParam = require("../../api/filesystem/UserParam"); -const config = require("../../config"); -const { chkperm, validate_fsentry_name } = require("../../helpers"); -const { TeePromise } = require("@heyputer/putility").libs.promise; -const { pausing_tee, logging_stream, offset_write_stream, stream_to_the_void } = require("../../util/streamutil"); -const { TYPE_DIRECTORY } = require("../FSNodeContext"); -const { LLRead } = require("../ll_operations/ll_read"); -const { RootNodeSelector, NodePathSelector } = require("../node/selectors"); -const { is_valid_node_name } = require("../validation"); -const { HLFilesystemOperation } = require("./definitions"); -const { MkTree } = require("./hl_mkdir"); -const { Actor } = require("../../services/auth/Actor"); -const { LLCWrite, LLOWrite } = require("../ll_operations/ll_write"); +const APIError = require('../../api/APIError'); +const FSNodeParam = require('../../api/filesystem/FSNodeParam'); +const FlagParam = require('../../api/filesystem/FlagParam'); +const StringParam = require('../../api/filesystem/StringParam'); +const UserParam = require('../../api/filesystem/UserParam'); +const config = require('../../config'); +const { chkperm, validate_fsentry_name } = require('../../helpers'); +const { TeePromise } = require('@heyputer/putility').libs.promise; +const { pausing_tee, logging_stream, offset_write_stream, stream_to_the_void } = require('../../util/streamutil'); +const { TYPE_DIRECTORY } = require('../FSNodeContext'); +const { LLRead } = require('../ll_operations/ll_read'); +const { RootNodeSelector, NodePathSelector } = require('../node/selectors'); +const { is_valid_node_name } = require('../validation'); +const { HLFilesystemOperation } = require('./definitions'); +const { MkTree } = require('./hl_mkdir'); +const { Actor } = require('../../services/auth/Actor'); +const { LLCWrite, LLOWrite } = require('../ll_operations/ll_write'); class WriteCommonFeature { install_in_instance (instance) { @@ -43,7 +43,7 @@ class WriteCommonFeature { ) { throw APIError.create('file_too_large', null, { max_size: config.max_file_size, - }) + }); } if ( @@ -52,9 +52,9 @@ class WriteCommonFeature { ) { throw APIError.create('thumbnail_too_large', null, { max_size: config.max_thumbnail_size, - }) + }); } - } + }; instance._verify_room = async function () { if ( ! this.values.file ) return; @@ -67,10 +67,10 @@ class WriteCommonFeature { const usage = await sizeService.get_usage(user.id); const capacity = await sizeService.get_storage_capacity(user.id); - if( capacity - usage - file.size < 0 ) { + if ( capacity - usage - file.size < 0 ) { throw APIError.create('storage_limit_reached'); } - } + }; } } @@ -85,11 +85,11 @@ class HLWrite extends HLFilesystemOperation { - deduplicate files with the same name // - create thumbnails; this will happen in low-level operation for now - create shortcuts - ` + `; static FEATURES = [ new WriteCommonFeature(), - ] + ]; static PARAMETERS = { // the parent directory, or a filepath that doesn't exist yet @@ -117,7 +117,7 @@ class HLWrite extends HLFilesystemOperation { static MODULES = { _path: require('path'), mime: require('mime-types'), - } + }; async _run () { const { context, values } = this; @@ -134,7 +134,7 @@ class HLWrite extends HLFilesystemOperation { this.checkpoint('before parent exists check'); - if ( ! await parent.exists() && values.create_missing_parents ) { + if ( !await parent.exists() && values.create_missing_parents ) { if ( ! (parent.selector instanceof NodePathSelector) ) { throw APIError.create('dest_does_not_exist', null, { parent: parent.selector, @@ -178,7 +178,7 @@ class HLWrite extends HLFilesystemOperation { this.checkpoint('check parent DNE or is not a directory'); if ( - ! await parent.exists() || + !await parent.exists() || await parent.get('type') !== TYPE_DIRECTORY ) { destination = parent; @@ -219,16 +219,16 @@ class HLWrite extends HLFilesystemOperation { const dest_exists = await destination.exists(); - if ( values.offset !== undefined && ! dest_exists ) { + if ( values.offset !== undefined && !dest_exists ) { throw APIError.create('offset_without_existing_file'); } - + // The correct ACL check here depends on context. // ll_write checks ACL, but we need to shortcut it here // or else we might send the user too much information. { const node_to_check = - ( dest_exists && overwrite && ! dedupe_name ) + ( dest_exists && overwrite && !dedupe_name ) ? destination : parent; const actor = values.actor ?? Actor.adapt(values.user); @@ -239,17 +239,16 @@ class HLWrite extends HLFilesystemOperation { } if ( dest_exists ) { - if ( ! overwrite && ! dedupe_name ) { + if ( !overwrite && !dedupe_name ) { throw APIError.create('item_with_same_name_exists', null, { - entry_name: target_name + entry_name: target_name, }); } if ( dedupe_name ) { - const fsEntryFetcher = context.get('services').get('fsEntryFetcher'); const target_ext = _path.extname(target_name); const target_noext = _path.basename(target_name, target_ext); - for ( let i=1 ;; i++ ) { + for ( let i = 1 ;; i++ ) { const try_new_name = `${target_noext} (${i})${target_ext}`; const exists = await parent.hasChild(try_new_name); if ( ! exists ) { @@ -302,53 +301,55 @@ class HLWrite extends HLFilesystemOperation { let thumbnail_promise = new TeePromise(); if ( await parent.isAppDataDirectory() || values.no_thumbnail ) { thumbnail_promise.resolve(undefined); - } else (async () => { - const reason = await (async () => { - const { mime } = this.modules; - const thumbnails = context.get('services').get('thumbnails'); - if ( values.thumbnail ) return 'already thumbnail'; - - const content_type = mime.contentType(target_name); - this.log.debug('CONTENT TYPE', content_type); - if ( ! content_type ) return 'no content type'; - if ( ! thumbnails.is_supported_mimetype(content_type) ) return 'unsupported content type'; - if ( ! thumbnails.is_supported_size(values.file.size) ) return 'too large'; - - // Create file object for thumbnail by either using an existing - // buffer (ex: /download endpoint) or by forking a stream - // (ex: /write and /batch endpoints). - const thumb_file = (() => { - if ( values.file.buffer ) return values.file; - - const [replace_stream, thumbnail_stream] = - pausing_tee(values.file.stream, 2); - - values.file.stream = replace_stream; - return { ...values.file, stream: thumbnail_stream }; - })(); - - let thumbnail; - try { - thumbnail = await thumbnails.thumbify(thumb_file); - } catch (e) { - stream_to_the_void(thumb_file.stream); - return 'thumbnail error: ' + e.message; - } + } else { + (async () => { + const reason = await (async () => { + const { mime } = this.modules; + const thumbnails = context.get('services').get('thumbnails'); + if ( values.thumbnail ) return 'already thumbnail'; + + const content_type = mime.contentType(target_name); + this.log.debug('CONTENT TYPE', content_type); + if ( ! content_type ) return 'no content type'; + if ( ! thumbnails.is_supported_mimetype(content_type) ) return 'unsupported content type'; + if ( ! thumbnails.is_supported_size(values.file.size) ) return 'too large'; + + // Create file object for thumbnail by either using an existing + // buffer (ex: /download endpoint) or by forking a stream + // (ex: /write and /batch endpoints). + const thumb_file = (() => { + if ( values.file.buffer ) return values.file; + + const [replace_stream, thumbnail_stream] = + pausing_tee(values.file.stream, 2); + + values.file.stream = replace_stream; + return { ...values.file, stream: thumbnail_stream }; + })(); + + let thumbnail; + try { + thumbnail = await thumbnails.thumbify(thumb_file); + } catch (e) { + stream_to_the_void(thumb_file.stream); + return `thumbnail error: ${ e.message}`; + } - const thumbnailData = { url: thumbnail } - if (thumbnailData.url) { - await svc_event.emit('thumbnail.created', thumbnailData); // An extension can modify where this thumbnail is stored - } + const thumbnailData = { url: thumbnail }; + if ( thumbnailData.url ) { + await svc_event.emit('thumbnail.created', thumbnailData); // An extension can modify where this thumbnail is stored + } - thumbnail_promise.resolve(thumbnailData.url); - })(); - if ( reason ) { - this.log.debug('REASON', reason); - thumbnail_promise.resolve(undefined); + thumbnail_promise.resolve(thumbnailData.url); + })(); + if ( reason ) { + this.log.debug('REASON', reason); + thumbnail_promise.resolve(undefined); // values.file.stream = logging_stream(values.file.stream); - } - })(); + } + })(); + } this.checkpoint('before delegate'); diff --git a/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js b/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js deleted file mode 100644 index 1de724a7b1..0000000000 --- a/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { DB_READ } = require("../../services/database/consts"); -const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../../filesystem/node/selectors"); -const BaseService = require("../../services/BaseService"); - -/** - * Service for fetching filesystem entries from the database using various selector types. - * Handles different methods of locating files and directories in the filesystem. - */ -module.exports = class DatabaseFSEntryFetcher extends BaseService { - static CONCERN = 'filesystem'; - - /** - * Initializes the default properties that will be selected from the database. - */ - _construct () { - this.defaultProperties = [ - 'id', - 'associated_app_id', - 'uuid', - 'public_token', - 'bucket', - 'bucket_region', - 'file_request_token', - 'user_id', - 'parent_uid', - 'is_dir', - 'is_public', - 'is_shortcut', - 'is_symlink', - 'symlink_path', - 'shortcut_to', - 'sort_by', - 'sort_order', - 'immutable', - 'name', - 'metadata', - 'modified', - 'created', - 'accessed', - 'size', - 'layout', - 'path', - ] - } - - /** - * Initializes the database connection for filesystem operations. - */ - _init () { - this.db = this.services.get('database').get(DB_READ, 'filesystem'); - } - - /** - * Finds a filesystem entry using the provided selector. - * @param {Object} selector - The selector object specifying how to find the entry - * @param {Object} fetch_entry_options - Options for fetching the entry - * @returns {Promise} The filesystem entry or null if not found - */ - async find (selector, fetch_entry_options) { - if ( selector instanceof RootNodeSelector ) { - return selector.entry; - } - if ( selector instanceof NodePathSelector ) { - return await this.findByPath( - selector.value, fetch_entry_options); - } - if ( selector instanceof NodeUIDSelector ) { - return await this.findByUID( - selector.value, fetch_entry_options); - } - if ( selector instanceof NodeInternalIDSelector ) { - return await this.findByID( - selector.id, fetch_entry_options); - } - if ( selector instanceof NodeChildSelector ) { - let id; - - if ( selector.parent instanceof RootNodeSelector ) { - id = await this.findNameInRoot(selector.name); - } else { - const parentEntry = await this.find(selector.parent); - if ( ! parentEntry ) return null; - id = await this.findNameInParent( - parentEntry.uuid, selector.name - ); - } - - if ( id === undefined ) return null; - if ( typeof id !== 'number' ) { - throw new Error( - 'unexpected type for id value', - typeof id, - id - ); - } - return this.find(new NodeInternalIDSelector('mysql', id)); - } - } - - /** - * Finds a filesystem entry by its UUID. - * @param {string} uuid - The UUID of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag - * @returns {Promise} The filesystem entry or undefined if not found - */ - async findByUID(uuid, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - let fsentry = await this.db.tryHardRead( - `SELECT ` + - this.defaultProperties.join(', ') + - (thumbnail ? `, thumbnail` : '') + - ` FROM fsentries WHERE uuid = ? LIMIT 1`, - [uuid] - ); - - return fsentry[0]; - } - - /** - * Finds a filesystem entry by its internal database ID. - * @param {number} id - The internal ID of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag - * @returns {Promise} The filesystem entry or undefined if not found - */ - async findByID(id, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - let fsentry = await this.db.tryHardRead( - `SELECT ` + - this.defaultProperties.join(', ') + - (thumbnail ? `, thumbnail` : '') + - ` FROM fsentries WHERE id = ? LIMIT 1`, - [id] - ); - - return fsentry[0]; - } - - /** - * Finds a filesystem entry by its full path. - * @param {string} path - The full path of the entry to find - * @param {Object} fetch_entry_options - Options including thumbnail flag and tracer - * @returns {Promise} The filesystem entry or false if not found - */ - async findByPath(path, fetch_entry_options = {}) { - const { thumbnail } = fetch_entry_options; - - if ( path === '/' ) { - return this.find(new RootNodeSelector()); - } - - const parts = path.split('/').filter(path => path !== ''); - if ( parts.length === 0 ) { - // TODO: invalid path; this should be an error - return false; - } - - - // TODO: use a closure table for more efficient path resolving - let parent_uid = null; - let result; - - const resultColsSql = this.defaultProperties.join(', ') + - (thumbnail ? `, thumbnail` : ''); - - result = await this.db.read( - `SELECT ` + resultColsSql + - ` FROM fsentries WHERE path=? LIMIT 1`, - [path] - ); - - // using knex instead - - if ( result[0] ) return result[0]; - - this.log.debug(`findByPath (not cached): ${path}`) - - const loop = async () => { - for ( let i=0 ; i < parts.length ; i++ ) { - const part = parts[i]; - const isLast = i == parts.length - 1; - const colsSql = isLast ? resultColsSql : 'uuid'; - if ( parent_uid === null ) { - result = await this.db.read( - `SELECT ` + colsSql + - ` FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`, - [part] - ); - } else { - result = await this.db.read( - `SELECT ` + colsSql + - ` FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`, - [parent_uid, part] - ); - } - - if ( ! result[0] ) return false; - parent_uid = result[0].uuid; - } - } - - if ( fetch_entry_options.tracer ) { - const tracer = fetch_entry_options.tracer; - const options = fetch_entry_options.trace_options; - await tracer.startActiveSpan(`fs:sql:findByPath`, - ...(options ? [options] : []), - async span => { - await loop(); - span.end(); - }); - } else { - await loop(); - } - - return result[0]; - } - - /** - * Finds the ID of a child entry with the given name in the root directory. - * @param {string} name - The name of the child entry to find - * @returns {Promise} The ID of the child entry or undefined if not found - */ - async findNameInRoot (name) { - let child_id = await this.db.read( - "SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1", - [name] - ); - return child_id[0]?.id; - } - - /** - * Finds the ID of a child entry with the given name under a specific parent. - * @param {string} parent_uid - The UUID of the parent directory - * @param {string} name - The name of the child entry to find - * @returns {Promise} The ID of the child entry or undefined if not found - */ - async findNameInParent (parent_uid, name) { - let child_id = await this.db.read( - "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1", - [parent_uid, name] - ); - return child_id[0]?.id; - } - - /** - * Checks if an entry with the given name exists under a specific parent. - * @param {string} parent_uid - The UUID of the parent directory - * @param {string} name - The name to check for - * @returns {Promise} True if the name exists under the parent, false otherwise - */ - async nameExistsUnderParent (parent_uid, name) { - let check_dupe = await this.db.read( - "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1", - [parent_uid, name] - ); - return !! check_dupe[0]; - } - - /** - * Checks if an entry with the given name exists under a parent specified by ID. - * @param {number} parent_id - The internal ID of the parent directory - * @param {string} name - The name to check for - * @returns {Promise} True if the name exists under the parent, false otherwise - */ - async nameExistsUnderParentID (parent_id, name) { - const parent = await this.findByID(parent_id); - if ( ! parent ) { - return false; - } - return this.nameExistsUnderParent(parent.uuid, name); - } -} diff --git a/src/backend/src/modules/puterfs/PuterFSModule.js b/src/backend/src/modules/puterfs/PuterFSModule.js index 45c1f8e349..14dd44e9e5 100644 --- a/src/backend/src/modules/puterfs/PuterFSModule.js +++ b/src/backend/src/modules/puterfs/PuterFSModule.js @@ -63,9 +63,6 @@ class PuterFSModule extends AdvancedBase { const { MountpointService } = require('./MountpointService'); services.registerService('mountpoint', MountpointService); - const DatabaseFSEntryFetcher = require('./DatabaseFSEntryFetcher'); - services.registerService('fsEntryFetcher', DatabaseFSEntryFetcher); - const { MemoryFSService } = require('./customfs/MemoryFSService'); services.registerService('memoryfs', MemoryFSService); }