From b881cb56558cd6845534cc9c8f9f72c0a6b3ab31 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 10 Feb 2025 13:46:19 +0900 Subject: [PATCH 1/8] feat: Impl noop storage backend --- src/ai/backend/common/defs.py | 3 + src/ai/backend/storage/config.py | 9 + src/ai/backend/storage/context.py | 1 + src/ai/backend/storage/noop/BUILD | 1 + src/ai/backend/storage/noop/__init__.py | 251 ++++++++++++++++++++++++ 5 files changed, 265 insertions(+) create mode 100644 src/ai/backend/storage/noop/BUILD create mode 100644 src/ai/backend/storage/noop/__init__.py diff --git a/src/ai/backend/common/defs.py b/src/ai/backend/common/defs.py index f3ae5ad7972..6c31c0e4732 100644 --- a/src/ai/backend/common/defs.py +++ b/src/ai/backend/common/defs.py @@ -44,3 +44,6 @@ DEFAULT_VFOLDER_PERMISSION_MODE: Final[int] = 0o755 VFOLDER_GROUP_PERMISSION_MODE: Final[int] = 0o775 + +NOOP_STORAGE_VOLUME_NAME: Final[str] = "noop" +NOOP_STORAGE_BACKEND_TYPE: Final[str] = "noop" diff --git a/src/ai/backend/storage/config.py b/src/ai/backend/storage/config.py index 240416d89ff..9e5d3685713 100644 --- a/src/ai/backend/storage/config.py +++ b/src/ai/backend/storage/config.py @@ -15,6 +15,7 @@ override_with_env, read_from_file, ) +from ai.backend.common.defs import NOOP_STORAGE_BACKEND_TYPE, NOOP_STORAGE_VOLUME_NAME from ai.backend.common.etcd import AsyncEtcd, ConfigScopes from ai.backend.logging.config import logging_config_iv @@ -139,6 +140,14 @@ def load_local_config(config_path: Path | None, debug: bool = False) -> dict[str if debug: override_key(raw_cfg, ("debug", "enabled"), True) + if NOOP_STORAGE_VOLUME_NAME in raw_cfg["volume"]: + raise ValueError(f"Volume name {NOOP_STORAGE_VOLUME_NAME} is not allowed") + + raw_cfg["volume"][NOOP_STORAGE_VOLUME_NAME] = { + "path": ".", + "backend": NOOP_STORAGE_BACKEND_TYPE, + } + try: local_config = check(raw_cfg, local_config_iv) local_config["_src"] = cfg_src_path diff --git a/src/ai/backend/storage/context.py b/src/ai/backend/storage/context.py index 112a31b5aed..6471e22fe3a 100644 --- a/src/ai/backend/storage/context.py +++ b/src/ai/backend/storage/context.py @@ -65,6 +65,7 @@ CephFSVolume.name: CephFSVolume, VASTVolume.name: VASTVolume, EXAScalerFSVolume.name: EXAScalerFSVolume, + NoopVolume.name: NoopVolume, } diff --git a/src/ai/backend/storage/noop/BUILD b/src/ai/backend/storage/noop/BUILD new file mode 100644 index 00000000000..db46e8d6c97 --- /dev/null +++ b/src/ai/backend/storage/noop/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/src/ai/backend/storage/noop/__init__.py b/src/ai/backend/storage/noop/__init__.py new file mode 100644 index 00000000000..b587e5b7cd6 --- /dev/null +++ b/src/ai/backend/storage/noop/__init__.py @@ -0,0 +1,251 @@ +from collections.abc import Sequence +from pathlib import Path, PurePosixPath +from typing import Any, AsyncIterator, Optional + +from ai.backend.common.defs import DEFAULT_VFOLDER_PERMISSION_MODE, NOOP_STORAGE_BACKEND_TYPE +from ai.backend.common.types import BinarySize, HardwareMetadata, QuotaScopeID + +from ..abc import AbstractFSOpModel, AbstractQuotaModel, AbstractVolume +from ..types import ( + CapacityUsage, + DirEntry, + FSPerfMetric, + QuotaConfig, + QuotaUsage, + TreeUsage, + VFolderID, +) + + +class NoopQuotaModel(AbstractQuotaModel): + def __init__(self) -> None: + return + + def mangle_qspath(self, ref: VFolderID | QuotaScopeID | str | None) -> Path: + return Path() + + async def create_quota_scope( + self, + quota_scope_id: QuotaScopeID, + options: Optional[QuotaConfig] = None, + extra_args: Optional[dict[str, Any]] = None, + ) -> None: + raise NotImplementedError + + async def describe_quota_scope( + self, + quota_scope_id: QuotaScopeID, + ) -> Optional[QuotaUsage]: + raise NotImplementedError + + async def update_quota_scope( + self, + quota_scope_id: QuotaScopeID, + config: QuotaConfig, + ) -> None: + raise NotImplementedError + + async def unset_quota( + self, + quota_scope_id: QuotaScopeID, + ) -> None: + raise NotImplementedError + + async def delete_quota_scope( + self, + quota_scope_id: QuotaScopeID, + ) -> None: + raise NotImplementedError + + +class NoopFSOpModel(AbstractFSOpModel): + def __init__(self) -> None: + return + + async def copy_tree( + self, + src_path: Path, + dst_path: Path, + ) -> None: + raise NotImplementedError + + async def move_tree( + self, + src_path: Path, + dst_path: Path, + ) -> None: + raise NotImplementedError + + async def delete_tree( + self, + path: Path, + ) -> None: + raise NotImplementedError + + def scan_tree( + self, + path: Path, + *, + recursive: bool = True, + ) -> AsyncIterator[DirEntry]: + raise NotImplementedError + + async def scan_tree_usage( + self, + path: Path, + ) -> TreeUsage: + raise NotImplementedError + + async def scan_tree_size( + self, + path: Path, + ) -> BinarySize: + raise NotImplementedError + + +class NoopVolume(AbstractVolume): + name = NOOP_STORAGE_BACKEND_TYPE + + async def create_quota_model(self) -> AbstractQuotaModel: + return NoopQuotaModel() + + async def create_fsop_model(self) -> AbstractFSOpModel: + return NoopFSOpModel() + + # ------ volume operations ------- + + async def get_capabilities(self) -> frozenset[str]: + return frozenset() + + async def get_hwinfo(self) -> HardwareMetadata: + return { + "status": "healthy", + "status_info": None, + "metadata": {}, + } + + async def create_vfolder( + self, + vfid: VFolderID, + exist_ok: bool = False, + mode: int = DEFAULT_VFOLDER_PERMISSION_MODE, + ) -> None: + return None + + async def delete_vfolder(self, vfid: VFolderID) -> None: + return None + + async def clone_vfolder( + self, + src_vfid: VFolderID, + dst_vfid: VFolderID, + ) -> None: + return None + + async def get_vfolder_mount(self, vfid: VFolderID, subpath: str) -> Path: + return Path() + + async def put_metadata(self, vfid: VFolderID, payload: bytes) -> None: + raise NotImplementedError + + async def get_metadata(self, vfid: VFolderID) -> bytes: + raise NotImplementedError + + async def get_performance_metric(self) -> FSPerfMetric: + raise NotImplementedError + + async def get_fs_usage(self) -> CapacityUsage: + return CapacityUsage(0, 0) + + async def get_usage( + self, + vfid: VFolderID, + relpath: PurePosixPath = PurePosixPath("."), + ) -> TreeUsage: + return TreeUsage(0, 0) + + async def get_used_bytes(self, vfid: VFolderID) -> BinarySize: + return BinarySize(0) + + # ------ vfolder operations ------- + + def scandir( + self, + vfid: VFolderID, + relpath: PurePosixPath, + *, + recursive: bool = True, + ) -> AsyncIterator[DirEntry]: + raise NotImplementedError + + async def mkdir( + self, + vfid: VFolderID, + relpath: PurePosixPath, + *, + parents: bool = False, + exist_ok: bool = False, + ) -> None: + raise NotImplementedError + + async def rmdir( + self, + vfid: VFolderID, + relpath: PurePosixPath, + *, + recursive: bool = False, + ) -> None: + raise NotImplementedError + + async def move_file( + self, + vfid: VFolderID, + src: PurePosixPath, + dst: PurePosixPath, + ) -> None: + raise NotImplementedError + + async def move_tree( + self, + vfid: VFolderID, + src: PurePosixPath, + dst: PurePosixPath, + ) -> None: + raise NotImplementedError + + async def copy_file( + self, + vfid: VFolderID, + src: PurePosixPath, + dst: PurePosixPath, + ) -> None: + raise NotImplementedError + + async def prepare_upload(self, vfid: VFolderID) -> str: + raise NotImplementedError + + async def add_file( + self, + vfid: VFolderID, + relpath: PurePosixPath, + payload: AsyncIterator[bytes], + ) -> None: + raise NotImplementedError + + def read_file( + self, + vfid: VFolderID, + relpath: PurePosixPath, + *, + chunk_size: int = 0, + ) -> AsyncIterator[bytes]: + raise NotImplementedError + + async def delete_files( + self, + vfid: VFolderID, + relpaths: Sequence[PurePosixPath], + *, + recursive: bool = False, + ) -> None: + raise NotImplementedError From b12b8d7026efe0fee0eda6a0c2dac958a1e88b5b Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 10 Feb 2025 14:20:14 +0900 Subject: [PATCH 2/8] add news fragment --- changes/3629.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/3629.feature.md diff --git a/changes/3629.feature.md b/changes/3629.feature.md new file mode 100644 index 00000000000..764f74fe594 --- /dev/null +++ b/changes/3629.feature.md @@ -0,0 +1 @@ +Impl noop storage backend From 7303e77004e2219827a39deb191c932097eb97c0 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 10 Feb 2025 14:33:15 +0900 Subject: [PATCH 3/8] update news fragment --- changes/3629.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/3629.feature.md b/changes/3629.feature.md index 764f74fe594..7815b2a9c0f 100644 --- a/changes/3629.feature.md +++ b/changes/3629.feature.md @@ -1 +1 @@ -Impl noop storage backend +Implement noop storage backend From 64e5e2658e398ac4773b4944bbb8a4bb325e3380 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Tue, 11 Feb 2025 23:36:13 +0900 Subject: [PATCH 4/8] resolve conflict --- src/ai/backend/storage/context.py | 1 + src/ai/backend/storage/{ => volumes}/noop/BUILD | 0 src/ai/backend/storage/{ => volumes}/noop/__init__.py | 4 ++-- 3 files changed, 3 insertions(+), 2 deletions(-) rename src/ai/backend/storage/{ => volumes}/noop/BUILD (100%) rename src/ai/backend/storage/{ => volumes}/noop/__init__.py (99%) diff --git a/src/ai/backend/storage/context.py b/src/ai/backend/storage/context.py index 6471e22fe3a..e8f10ce69ee 100644 --- a/src/ai/backend/storage/context.py +++ b/src/ai/backend/storage/context.py @@ -40,6 +40,7 @@ from .volumes.dellemc import DellEMCOneFSVolume from .volumes.gpfs import GPFSVolume from .volumes.netapp import NetAppVolume +from .volumes.noop import NoopVolume from .volumes.purestorage import FlashBladeVolume from .volumes.vast import VASTVolume from .volumes.vfs import BaseVolume diff --git a/src/ai/backend/storage/noop/BUILD b/src/ai/backend/storage/volumes/noop/BUILD similarity index 100% rename from src/ai/backend/storage/noop/BUILD rename to src/ai/backend/storage/volumes/noop/BUILD diff --git a/src/ai/backend/storage/noop/__init__.py b/src/ai/backend/storage/volumes/noop/__init__.py similarity index 99% rename from src/ai/backend/storage/noop/__init__.py rename to src/ai/backend/storage/volumes/noop/__init__.py index b587e5b7cd6..0f3a8725f60 100644 --- a/src/ai/backend/storage/noop/__init__.py +++ b/src/ai/backend/storage/volumes/noop/__init__.py @@ -5,8 +5,7 @@ from ai.backend.common.defs import DEFAULT_VFOLDER_PERMISSION_MODE, NOOP_STORAGE_BACKEND_TYPE from ai.backend.common.types import BinarySize, HardwareMetadata, QuotaScopeID -from ..abc import AbstractFSOpModel, AbstractQuotaModel, AbstractVolume -from ..types import ( +from ...types import ( CapacityUsage, DirEntry, FSPerfMetric, @@ -15,6 +14,7 @@ TreeUsage, VFolderID, ) +from ..abc import AbstractFSOpModel, AbstractQuotaModel, AbstractVolume class NoopQuotaModel(AbstractQuotaModel): From d382f18ad9b772dbc86d74c340fa6b9d1267d640 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 17 Feb 2025 14:14:10 +0900 Subject: [PATCH 5/8] Do not insert noop storage volume --- src/ai/backend/storage/config.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/ai/backend/storage/config.py b/src/ai/backend/storage/config.py index 9e5d3685713..240416d89ff 100644 --- a/src/ai/backend/storage/config.py +++ b/src/ai/backend/storage/config.py @@ -15,7 +15,6 @@ override_with_env, read_from_file, ) -from ai.backend.common.defs import NOOP_STORAGE_BACKEND_TYPE, NOOP_STORAGE_VOLUME_NAME from ai.backend.common.etcd import AsyncEtcd, ConfigScopes from ai.backend.logging.config import logging_config_iv @@ -140,14 +139,6 @@ def load_local_config(config_path: Path | None, debug: bool = False) -> dict[str if debug: override_key(raw_cfg, ("debug", "enabled"), True) - if NOOP_STORAGE_VOLUME_NAME in raw_cfg["volume"]: - raise ValueError(f"Volume name {NOOP_STORAGE_VOLUME_NAME} is not allowed") - - raw_cfg["volume"][NOOP_STORAGE_VOLUME_NAME] = { - "path": ".", - "backend": NOOP_STORAGE_BACKEND_TYPE, - } - try: local_config = check(raw_cfg, local_config_iv) local_config["_src"] = cfg_src_path From ac9babb08474fd6fde0e99d290434766de64ee28 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 17 Feb 2025 14:37:31 +0900 Subject: [PATCH 6/8] handle noop volume in storage context --- src/ai/backend/storage/context.py | 9 ++++++++- .../backend/storage/volumes/noop/__init__.py | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/ai/backend/storage/context.py b/src/ai/backend/storage/context.py index e8f10ce69ee..2836a0d41fd 100644 --- a/src/ai/backend/storage/context.py +++ b/src/ai/backend/storage/context.py @@ -16,6 +16,7 @@ from aiohttp import web from aiohttp.typedefs import Middleware +from ai.backend.common.defs import NOOP_STORAGE_VOLUME_NAME from ai.backend.common.etcd import AsyncEtcd from ai.backend.common.events import ( EventDispatcher, @@ -40,7 +41,7 @@ from .volumes.dellemc import DellEMCOneFSVolume from .volumes.gpfs import GPFSVolume from .volumes.netapp import NetAppVolume -from .volumes.noop import NoopVolume +from .volumes.noop import NoopVolume, init_noop_volume from .volumes.purestorage import FlashBladeVolume from .volumes.vast import VASTVolume from .volumes.vfs import BaseVolume @@ -188,6 +189,12 @@ async def __aexit__(self, *exc_info) -> Optional[bool]: @actxmgr async def get_volume(self, name: str) -> AsyncIterator[AbstractVolume]: + if name == NOOP_STORAGE_VOLUME_NAME: + noop_volume_obj = init_noop_volume( + self.etcd, self.event_dispatcher, self.event_producer + ) + yield noop_volume_obj + return if name in self.volumes: yield self.volumes[name] else: diff --git a/src/ai/backend/storage/volumes/noop/__init__.py b/src/ai/backend/storage/volumes/noop/__init__.py index 0f3a8725f60..b65bcac5938 100644 --- a/src/ai/backend/storage/volumes/noop/__init__.py +++ b/src/ai/backend/storage/volumes/noop/__init__.py @@ -3,6 +3,8 @@ from typing import Any, AsyncIterator, Optional from ai.backend.common.defs import DEFAULT_VFOLDER_PERMISSION_MODE, NOOP_STORAGE_BACKEND_TYPE +from ai.backend.common.etcd import AsyncEtcd +from ai.backend.common.events import EventDispatcher, EventProducer from ai.backend.common.types import BinarySize, HardwareMetadata, QuotaScopeID from ...types import ( @@ -249,3 +251,19 @@ async def delete_files( recursive: bool = False, ) -> None: raise NotImplementedError + + +def init_noop_volume( + etcd: AsyncEtcd, + event_dispatcher: EventDispatcher, + event_producer: EventProducer, +) -> NoopVolume: + return NoopVolume( + {}, + Path(), + etcd=etcd, + event_dispatcher=event_dispatcher, + event_producer=event_producer, + watcher=None, + options=None, + ) From b1d75fe704eb346e432f924e21cabef3b3531f1b Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Mon, 17 Feb 2025 23:53:20 +0900 Subject: [PATCH 7/8] return empty values rather than raising NotImplementedError --- .../backend/storage/volumes/noop/__init__.py | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/src/ai/backend/storage/volumes/noop/__init__.py b/src/ai/backend/storage/volumes/noop/__init__.py index b65bcac5938..8879e40a8a1 100644 --- a/src/ai/backend/storage/volumes/noop/__init__.py +++ b/src/ai/backend/storage/volumes/noop/__init__.py @@ -1,4 +1,5 @@ from collections.abc import Sequence +from datetime import datetime from pathlib import Path, PurePosixPath from typing import Any, AsyncIterator, Optional @@ -10,18 +11,26 @@ from ...types import ( CapacityUsage, DirEntry, + DirEntryType, FSPerfMetric, QuotaConfig, QuotaUsage, + Stat, TreeUsage, VFolderID, ) from ..abc import AbstractFSOpModel, AbstractQuotaModel, AbstractVolume +async def _return_empty_dir_entry() -> AsyncIterator[DirEntry]: + yield DirEntry( + "", Path(), DirEntryType.FILE, Stat(0, "", 0, datetime.now(), datetime.now()), "" + ) + + class NoopQuotaModel(AbstractQuotaModel): def __init__(self) -> None: - return + pass def mangle_qspath(self, ref: VFolderID | QuotaScopeID | str | None) -> Path: return Path() @@ -32,32 +41,32 @@ async def create_quota_scope( options: Optional[QuotaConfig] = None, extra_args: Optional[dict[str, Any]] = None, ) -> None: - raise NotImplementedError + pass async def describe_quota_scope( self, quota_scope_id: QuotaScopeID, ) -> Optional[QuotaUsage]: - raise NotImplementedError + pass async def update_quota_scope( self, quota_scope_id: QuotaScopeID, config: QuotaConfig, ) -> None: - raise NotImplementedError + pass async def unset_quota( self, quota_scope_id: QuotaScopeID, ) -> None: - raise NotImplementedError + pass async def delete_quota_scope( self, quota_scope_id: QuotaScopeID, ) -> None: - raise NotImplementedError + pass class NoopFSOpModel(AbstractFSOpModel): @@ -69,20 +78,20 @@ async def copy_tree( src_path: Path, dst_path: Path, ) -> None: - raise NotImplementedError + pass async def move_tree( self, src_path: Path, dst_path: Path, ) -> None: - raise NotImplementedError + pass async def delete_tree( self, path: Path, ) -> None: - raise NotImplementedError + pass def scan_tree( self, @@ -90,19 +99,19 @@ def scan_tree( *, recursive: bool = True, ) -> AsyncIterator[DirEntry]: - raise NotImplementedError + return _return_empty_dir_entry() async def scan_tree_usage( self, path: Path, ) -> TreeUsage: - raise NotImplementedError + return TreeUsage(0, 0) async def scan_tree_size( self, path: Path, ) -> BinarySize: - raise NotImplementedError + return BinarySize(0) class NoopVolume(AbstractVolume): @@ -148,13 +157,13 @@ async def get_vfolder_mount(self, vfid: VFolderID, subpath: str) -> Path: return Path() async def put_metadata(self, vfid: VFolderID, payload: bytes) -> None: - raise NotImplementedError + pass async def get_metadata(self, vfid: VFolderID) -> bytes: - raise NotImplementedError + return b"" async def get_performance_metric(self) -> FSPerfMetric: - raise NotImplementedError + return FSPerfMetric(0, 0, 0, 0, 0.0, 0.0) async def get_fs_usage(self) -> CapacityUsage: return CapacityUsage(0, 0) @@ -178,7 +187,7 @@ def scandir( *, recursive: bool = True, ) -> AsyncIterator[DirEntry]: - raise NotImplementedError + return _return_empty_dir_entry() async def mkdir( self, @@ -188,7 +197,7 @@ async def mkdir( parents: bool = False, exist_ok: bool = False, ) -> None: - raise NotImplementedError + pass async def rmdir( self, @@ -197,7 +206,7 @@ async def rmdir( *, recursive: bool = False, ) -> None: - raise NotImplementedError + pass async def move_file( self, @@ -205,7 +214,7 @@ async def move_file( src: PurePosixPath, dst: PurePosixPath, ) -> None: - raise NotImplementedError + pass async def move_tree( self, @@ -213,7 +222,7 @@ async def move_tree( src: PurePosixPath, dst: PurePosixPath, ) -> None: - raise NotImplementedError + pass async def copy_file( self, @@ -221,10 +230,10 @@ async def copy_file( src: PurePosixPath, dst: PurePosixPath, ) -> None: - raise NotImplementedError + pass async def prepare_upload(self, vfid: VFolderID) -> str: - raise NotImplementedError + return "" async def add_file( self, @@ -232,7 +241,7 @@ async def add_file( relpath: PurePosixPath, payload: AsyncIterator[bytes], ) -> None: - raise NotImplementedError + pass def read_file( self, @@ -241,7 +250,10 @@ def read_file( *, chunk_size: int = 0, ) -> AsyncIterator[bytes]: - raise NotImplementedError + async def _noop() -> AsyncIterator[bytes]: + yield b"" + + return _noop() async def delete_files( self, @@ -250,7 +262,7 @@ async def delete_files( *, recursive: bool = False, ) -> None: - raise NotImplementedError + pass def init_noop_volume( From a89ce6bbeeca2b43ed390c980badf1706644d720 Mon Sep 17 00:00:00 2001 From: Sanghun Lee Date: Fri, 21 Feb 2025 13:50:54 +0900 Subject: [PATCH 8/8] insert noop volume in initialize step --- src/ai/backend/storage/context.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ai/backend/storage/context.py b/src/ai/backend/storage/context.py index 2836a0d41fd..909739a1d59 100644 --- a/src/ai/backend/storage/context.py +++ b/src/ai/backend/storage/context.py @@ -122,7 +122,11 @@ def __init__( dsn: Optional[str] = None, metric_registry: CommonMetricRegistry = CommonMetricRegistry.instance(), ) -> None: - self.volumes = {} + self.volumes = { + NOOP_STORAGE_VOLUME_NAME: init_noop_volume( + self.etcd, self.event_dispatcher, self.event_producer + ) + } self.pid = pid self.pidx = pidx self.node_id = node_id @@ -189,12 +193,6 @@ async def __aexit__(self, *exc_info) -> Optional[bool]: @actxmgr async def get_volume(self, name: str) -> AsyncIterator[AbstractVolume]: - if name == NOOP_STORAGE_VOLUME_NAME: - noop_volume_obj = init_noop_volume( - self.etcd, self.event_dispatcher, self.event_producer - ) - yield noop_volume_obj - return if name in self.volumes: yield self.volumes[name] else: