Merge pull request #11 from londogard/copilot/implement-s3-import-command

fix: WIP

Author: Lundez

README.md

@@ -109,8 +109,16 @@ uv run fluxel commit --repo /tmp/fluxel-demo -m "update"
 
 uv run fluxel branch --repo /tmp/fluxel-demo experiment
 uv run fluxel diff --repo /tmp/fluxel-demo <from_ref> <to_ref>
-uv run fluxel rm --repo /tmp/fluxel-demo old-prefix -m "remove old files"
-uv run fluxel mv --repo /tmp/fluxel-demo raw/images curated/images -m "rename image prefix"
+
+# Stage and commit metadata mutations
+uv run fluxel rm --repo /tmp/fluxel-demo old-prefix
+uv run fluxel mv --repo /tmp/fluxel-demo raw/images curated/images
+uv run fluxel status --repo /tmp/fluxel-demo
+uv run fluxel commit --repo /tmp/fluxel-demo -m "clean up old files and rename image prefix"
+
+# Or commit metadata mutations directly with a message
+uv run fluxel rm --repo /tmp/fluxel-demo logs/2025 -m "remove old logs"
+uv run fluxel mv --repo /tmp/fluxel-demo incoming/images curated/images -m "reorganize images"
 
 # remote repo metadata operations from the current working tree
 uv run fluxel branch --repo s3://my-bucket/datasets/demo feature

scripts/run_s3_integration.sh

@@ -2,9 +2,11 @@
 from .core import (
     DEFAULT_CHUNK_SIZE,
     AnalyticalIndexPaths,
+    BlobTransferBackend,
     LocalClientState,
     CommitObject,
     DiffEntry,
+    FileEntry,
     FluxelFileSystem,
     FluxelLayout,
     FluxelRepository,
@@ -18,9 +20,10 @@
     ManifestReader,
     ManifestWriter,
     OptimisticLockError,
-    RepositoryStore,
     RefConflictError,
     RemoveResult,
+    S3BlobTransferBackend,
+    S5CmdBlobTransferBackend,
     S3RepositoryStore,
     S3StorageBackend,
     StorageBackend,
@@ -34,6 +37,7 @@
     build_analytical_index,
     blob_relpath,
     branch,
+    build_blob_transfer_backend,
     build_manifest_entries,
     commit,
     diff,
@@ -53,18 +57,22 @@
 __all__ = [
     "DEFAULT_CHUNK_SIZE",
     "AnalyticalIndexPaths",
+    "BlobTransferBackend",
     "LocalClientState",
     "CommitObject",
     "DiffEntry",
     "FluxelFileSystem",
     "FluxelLayout",
     "FluxelRepository",
     "FluxelURI",
+    "FileEntry",
     "BranchRefState",
     "MoveResult",
     "open_repository",
     "RefConflictError",
     "RemoveResult",
+    "S3BlobTransferBackend",
+    "S5CmdBlobTransferBackend",
     "StageChange",
     "StageStatus",
     "MergeResult",
@@ -85,6 +93,7 @@
     "blob_relpath",
     "add",
     "branch",
+    "build_blob_transfer_backend",
     "build_manifest_entries",
     "build_parser",
     "commit",

src/fluxel/__init__.py

@@ -19,15 +19,13 @@
     drop_analytical_index,
     import_s3,
     merge,
-    move,
+    move_staged,
     query_analytical_index,
-    remove,
     rm,
     status,
     verify,
 )
 
-
 IdentityMode = Literal["blake3", "meta"]
 HANDLED_CLI_ERRORS = (
     BotoCoreError,
@@ -56,6 +54,11 @@ class CommitArgs:
         help="Commit only staged changes for a branch",
     )
     ref: str | None = None  # Branch ref to update (defaults to current branch)
+    transfer_backend: str | None = field(
+        default=None,
+        alias=["--transfer-backend", "--backend"],
+        help="Blob transfer backend (boto3 or s5cmd)",
+    )
 
 
 @dataclass
@@ -79,6 +82,11 @@ class ImportArgs:
         help="Repository path or URI",
     )
     ref: str | None = None  # Branch ref to update (defaults to current branch)
+    transfer_backend: str | None = field(
+        default=None,
+        alias=["--transfer-backend", "--backend"],
+        help="Blob transfer backend (boto3 or s5cmd)",
+    )
 
 
 @dataclass
@@ -101,16 +109,16 @@ class AddArgs:
         default=None,
         help="Branch ref for staging (defaults to current branch)",
     )
+    transfer_backend: str | None = field(
+        default=None,
+        alias=["--transfer-backend", "--backend"],
+        help="Blob transfer backend (boto3 or s5cmd)",
+    )
 
 
 @dataclass
 class RmArgs:
     paths: list[str] = field(positional=True, nargs="+", help="Paths to remove")
-    message: str | None = field(
-        default=None,
-        alias=["-m", "--message"],
-        help="Commit message for a metadata-only removal",
-    )
     root: str = field(
         default=".",
         alias="--repo",
@@ -120,10 +128,6 @@ class RmArgs:
         default=None,
         help="Branch ref for staging (defaults to current branch)",
     )
-    staged: bool = flag(
-        False,
-        help="Stage removals instead of writing a metadata-only commit",
-    )
 
 
 @dataclass
@@ -133,15 +137,14 @@ class MoveArgs:
         positional=True,
         help="Destination path or prefix",
     )
-    message: str = field(alias=["-m", "--message"], help="Commit message")
     root: str = field(
         default=".",
         alias="--repo",
         help="Repository path or URI",
     )
     ref: str | None = field(
         default=None,
-        help="Branch ref to update (defaults to current branch)",
+        help="Branch ref for staging (defaults to current branch)",
     )
 
 
@@ -209,6 +212,11 @@ class VerifyArgs:
         alias="--dry-run",
         help="Report entries that would be verified without writing blobs or commits",
     )
+    transfer_backend: str | None = field(
+        default=None,
+        alias=["--transfer-backend", "--backend"],
+        help="Blob transfer backend (boto3 or s5cmd)",
+    )
 
 
 @dataclass
@@ -356,6 +364,7 @@ def run_cli(argv: list[str] | None = None) -> int:
                 identity_mode=command.identity,
                 staged=command.staged,
                 ref=command.ref,
+                blob_transfer=command.transfer_backend,
             )
             print(commit_id)
             return 0
@@ -368,6 +377,7 @@ def run_cli(argv: list[str] | None = None) -> int:
                 identity_mode=command.identity,
                 path_patterns=_flatten_option_values(command.path_patterns),
                 ref=command.ref,
+                blob_transfer=command.transfer_backend,
             )
             print(commit_id)
             return 0
@@ -379,60 +389,24 @@ def run_cli(argv: list[str] | None = None) -> int:
                 ref=command.ref,
                 identity_mode=command.identity,
                 destination_path=command.destination_path,
+                blob_transfer=command.transfer_backend,
             )
             print(json.dumps(_stage_payload(stage), indent=2))
             return 0
 
         if isinstance(command, RmArgs):
-            if command.message is not None and command.staged:
-                print(
-                    "rm error: cannot combine --message with --staged",
-                    file=sys.stderr,
-                )
-                return 2
-            if command.message is not None:
-                result = remove(
-                    root=command.root,
-                    paths=command.paths,
-                    message=command.message,
-                    ref=command.ref,
-                )
-                print(
-                    json.dumps(
-                        {
-                            "ref": result.ref,
-                            "commit_id": result.commit_id,
-                            "removed_paths": result.removed_paths,
-                        },
-                        indent=2,
-                    )
-                )
-                return 0
-
             stage = rm(root=command.root, paths=command.paths, ref=command.ref)
             print(json.dumps(_stage_payload(stage), indent=2))
             return 0
 
         if isinstance(command, MoveArgs):
-            result = move(
+            stage = move_staged(
                 root=command.root,
                 source_path=command.source_path,
                 destination_path=command.destination_path,
-                message=command.message,
                 ref=command.ref,
             )
-            print(
-                json.dumps(
-                    {
-                        "ref": result.ref,
-                        "commit_id": result.commit_id,
-                        "source_path": result.source_path,
-                        "destination_path": result.destination_path,
-                        "moved_paths": result.moved_paths,
-                    },
-                    indent=2,
-                )
-            )
+            print(json.dumps(_stage_payload(stage), indent=2))
             return 0
 
         if isinstance(command, StatusArgs):
@@ -493,6 +467,7 @@ def run_cli(argv: list[str] | None = None) -> int:
                 ref=command.ref,
                 path_prefixes=_flatten_option_values(command.path),
                 dry_run=command.dry_run,
+                blob_transfer=command.transfer_backend,
             )
             payload = {
                 "commit_id": result.commit_id,

src/fluxel/cli.py

@@ -9,6 +9,7 @@
 )
 from .layout import FluxelLayout, blob_relpath, initialize_fluxel_layout
 from .manifest import (
+    FileEntry,
     ManifestEntry,
     ManifestReader,
     ManifestWriter,
@@ -40,17 +41,22 @@
     import_s3,
     merge,
     move,
+    move_staged,
     remove,
     rm,
     status,
     verify,
 )
 from .storage import (
+    BlobTransferBackend,
     LocalStorageBackend,
     OptimisticLockError,
+    S3BlobTransferBackend,
+    S5CmdBlobTransferBackend,
     S3ObjectMetadata,
     S3StorageBackend,
     StorageBackend,
+    build_blob_transfer_backend,
     iter_s3_objects,
     open_source_uri,
     parse_s3_uri,
@@ -60,6 +66,7 @@
     "DEFAULT_CHUNK_SIZE",
     "AnalyticalIndexPaths",
     "LocalClientState",
+    "BlobTransferBackend",
     "CommitObject",
     "DiffEntry",
     "FluxelFileSystem",
@@ -69,11 +76,14 @@
     "open_repository",
     "RefConflictError",
     "RemoveResult",
+    "S3BlobTransferBackend",
+    "S5CmdBlobTransferBackend",
     "StageChange",
     "StageStatus",
     "VerifyResult",
     "FluxelURI",
     "FluxelLayout",
+    "FileEntry",
     "BranchRefState",
     "LocalRepositoryStore",
     "LocalStorageBackend",
@@ -85,25 +95,27 @@
     "S3RepositoryStore",
     "S3StorageBackend",
     "StorageBackend",
+    "build_analytical_index",
     "blake3_digest_file",
     "blake3_digest_stream",
-    "build_analytical_index",
-    "build_manifest_entries",
+    "build_blob_transfer_backend",
     "blob_relpath",
+    "build_manifest_entries",
     "add",
     "branch",
     "commit",
     "diff",
+    "drop_analytical_index",
     "import_s3",
+    "initialize_fluxel_layout",
     "merge",
     "move",
+    "move_staged",
+    "query_analytical_index",
     "remove",
     "rm",
     "status",
     "verify",
-    "drop_analytical_index",
-    "initialize_fluxel_layout",
-    "query_analytical_index",
     "S3ObjectMetadata",
     "walk_files",
     "iter_s3_objects",