Merge pull request #70 from RRosio/file_transfer

Update file upload/download handlers

Author: ericsnekbytes

Diff for: conftest.py

@@ -12,23 +12,37 @@
 ]
 
 
+@pytest.fixture(scope="function")
+def setup_tmp_local(tmp_path: Path):
+    local_root = tmp_path / "test"
+    local_root.mkdir()
+    local_file = local_root / "file_loc.txt"
+    local_file.touch()
+    local_empty_root = tmp_path / "empty"
+    local_empty_root.mkdir()
+
+    return [local_root, local_empty_root]
+
+
 @pytest.fixture(scope="function", autouse=True)
-def setup_config_file_fs(tmp_path: Path):
+def setup_config_file_fs(tmp_path: Path, setup_tmp_local):
+    tmp_local = setup_tmp_local[0]
+    empty_tmp_local = setup_tmp_local[1]
     config_dir = tmp_path / "config"
     config_dir.mkdir(exist_ok=True)
 
-    yaml_content = """sources:
+    yaml_content = f"""sources:
   - name: "TestSourceAWS"
     path: "s3://my-test-bucket/"
     additional_options:
       anon: false
       key: "my-access-key"
       secret: "my-secret-key"
   - name: "TestDir"
-    path: "/Users/someuser/Desktop/test_fsspec"
+    path: "{tmp_local}"
     protocol: "local"
   - name: "TestEmptyLocalDir"
-    path: "/Users/someuser/Desktop/notebooks/sample/nothinghere"
+    path: "{empty_tmp_local}"
     protocol: "local"
   - name: "TestMem Source"
     path: "/my_mem_dir"
@@ -50,31 +64,31 @@ def setup_config_file_fs(tmp_path: Path):
 def fs_manager_instance(setup_config_file_fs):
     fs_manager = setup_config_file_fs
     fs_info = fs_manager.get_filesystem_by_protocol("memory")
-    fs = fs_info["info"]["instance"]
+    mem_fs = fs_info["info"]["instance"]
     mem_root_path = fs_info["info"]["path"]
 
-    if fs:
-        if fs.exists(f"{mem_root_path}/test_dir"):
-            fs.rm(f"{mem_root_path}/test_dir", recursive=True)
-        if fs.exists(f"{mem_root_path}/second_dir"):
-            fs.rm(f"{mem_root_path}/second_dir", recursive=True)
+    if mem_fs:
+        if mem_fs.exists(f"{mem_root_path}/test_dir"):
+            mem_fs.rm(f"{mem_root_path}/test_dir", recursive=True)
+        if mem_fs.exists(f"{mem_root_path}/second_dir"):
+            mem_fs.rm(f"{mem_root_path}/second_dir", recursive=True)
 
-        fs.touch(f"{mem_root_path}/file_in_root.txt")
-        with fs.open(f"{mem_root_path}/file_in_root.txt", "wb") as f:
+        mem_fs.touch(f"{mem_root_path}/file_in_root.txt")
+        with mem_fs.open(f"{mem_root_path}/file_in_root.txt", "wb") as f:
             f.write("Root file content".encode())
 
-        fs.mkdir(f"{mem_root_path}/test_dir", exist_ok=True)
-        fs.mkdir(f"{mem_root_path}/second_dir", exist_ok=True)
-        # fs.mkdir(f'{mem_root_path}/second_dir/subdir', exist_ok=True)
-        fs.touch(f"{mem_root_path}/test_dir/file1.txt")
-        with fs.open(f"{mem_root_path}/test_dir/file1.txt", "wb") as f:
+        mem_fs.mkdir(f"{mem_root_path}/test_dir", exist_ok=True)
+        mem_fs.mkdir(f"{mem_root_path}/second_dir", exist_ok=True)
+        # mem_fs.mkdir(f'{mem_root_path}/second_dir/subdir', exist_ok=True)
+        mem_fs.touch(f"{mem_root_path}/test_dir/file1.txt")
+        with mem_fs.open(f"{mem_root_path}/test_dir/file1.txt", "wb") as f:
             f.write("Test content".encode())
             f.close()
     else:
         print("In memory filesystem NOT FOUND")
 
-    if fs.exists(f"{mem_root_path}/test_dir/file1.txt"):
-        file_info = fs.info(f"{mem_root_path}/test_dir/file1.txt")
+    if mem_fs.exists(f"{mem_root_path}/test_dir/file1.txt"):
+        file_info = mem_fs.info(f"{mem_root_path}/test_dir/file1.txt")
         print(f"File exists. size: {file_info}")
     else:
         print("File does not exist!")

Diff for: jupyter_fsspec/handlers.py

@@ -183,16 +183,21 @@ def post(self):
         try:
             if action == "upload":
                 # upload     fs_instance.put(local_path, remote_path)
-                fs_instance.put(item_path, destination)
-                response["description"] = f"Uploaded {item_path} to {destination}."
+                local_path = item_path
+                remote_path = destination
+                fs_instance.put(local_path, remote_path, recursive=True)
+                response["description"] = f"Uploaded {local_path} to {remote_path}."
             else:  # download
                 # download   fs_instance.get(remote_path, local_path)
-                fs_instance.get(destination, item_path)
-                response["description"] = f"Downloaded {destination} to {item_path}."
+                local_path = destination
+                remote_path = item_path
+                fs_instance.get(remote_path, local_path, recursive=True)
+                response["description"] = f"Downloaded {remote_path} to {local_path}."
 
             response["status"] = "success"
             self.set_status(200)
         except Exception as e:
+            print(f"Error uploading/downloading file: {e}")
             self.set_status(500)
             response["status"] = "failed"
             response["description"] = str(e)

Diff for: jupyter_fsspec/tests/test_api.py

@@ -478,25 +478,130 @@ async def xtest___async_s3_file_operations(mock_s3_fs):
     print(f"contents: {contents}")
 
 
-# TODO: Test transfer endpoint
-async def xtest_file_transfer(fs_manager_instance_parameterized, jp_fetch):
-    fs_manager = fs_manager_instance_parameterized
-    fs_info = fs_manager.get_filesystem_by_protocol("memory")
-    fs = fs_info["info"]["instance"]
-    assert fs is not None
+# TODO: Update file paths from memory to remote; Update remote_key usage
+async def xtest_file_transfer(fs_manager_instance, jp_fetch):
+    fs_manager = fs_manager_instance
+    remote_fs_info = fs_manager.get_filesystem_by_protocol("s3")
+    remote_key = remote_fs_info["key"]
+    remote_fs = remote_fs_info["info"]["instance"]
+    remote_item_path = remote_fs_info["info"]["path"]
+    assert remote_fs is not None
+    assert remote_fs.exists(remote_item_path)
+
+    local_fs_info = fs_manager.get_filesystem_by_protocol("local")
+    local_key = local_fs_info["key"]
+    local_fs = local_fs_info["info"]["instance"]
+    local_item_path = local_fs_info["info"]["path"]
+    assert local_fs is not None
+
+    # upload file [local to remote]
+    upload_filepath = f"{local_item_path}/file_loc.txt"
+    assert local_fs.exists(upload_filepath)
+    upload_file_payload = {
+        "item_path": upload_filepath,
+        "content": remote_item_path,
+        "destination_key": remote_key,
+        "action": "upload",
+    }
+    upload_file_res = await jp_fetch(
+        "jupyter_fsspec",
+        "files",
+        "transfer",
+        method="POST",
+        params={"key": local_key},
+        body=json.dumps(upload_file_payload),
+    )
+
+    upfile_body_json = upload_file_res.body.decode("utf-8")
+    upfile_body = json.loads(upfile_body_json)
+
+    assert upfile_body["status"] == "success"
+    assert upfile_body["description"] == f"Uploaded {upload_filepath} to /my_mem_dir."
+
+    uploaded_filepath = remote_item_path + "/file_loc.txt"
+    assert remote_fs.exists(uploaded_filepath)
+
+    # upload dir [local to remote]
+    upload_dirpath = local_item_path
+    upload_dir_payload = {
+        "item_path": upload_dirpath,
+        "content": remote_item_path,
+        "destination_key": remote_key,
+        "action": "upload",
+    }
+    upload_dir_res = await jp_fetch(
+        "jupyter_fsspec",
+        "files",
+        "transfer",
+        method="POST",
+        params={"key": remote_key},
+        body=json.dumps(upload_dir_payload),
+    )
+
+    updir_body_json = upload_dir_res.body.decode("utf-8")
+    updir_body = json.loads(updir_body_json)
+    assert updir_body["status"] == "success"
+    assert updir_body["description"] == f"Uploaded {upload_dirpath} to /my_mem_dir."
+
+    # download file [other to remote]
+    download_filepath = "/my_mem_dir/test_dir/file1.txt"
+    download_file_payload = {
+        "item_path": download_filepath,
+        "content": local_item_path,
+        "destination_key": local_key,
+        "action": "download",
+    }
+    download_file_res = await jp_fetch(
+        "jupyter_fsspec",
+        "files",
+        "transfer",
+        method="POST",
+        params={"key": remote_key},
+        body=json.dumps(download_file_payload),
+    )
 
-    # # copy file
-    # copy_filepath = f'{fs_root_path}/test_dir/file1.txt'
-    # copy_file_payload = {"item_path": copy_filepath, "content": "/my_local_dir/", "destination_key": "", "action": "copy"}
-    # copy_file_res = await jp_fetch("jupyter_fsspec", "files", "transfer", method="POST", params={"key": mem_key}, body=json.dumps(copy_file_payload))
+    download_file_body_json = download_file_res.body.decode("utf-8")
+    download_file_body = json.loads(download_file_body_json)
+    assert download_file_body["status"] == "success"
+    assert (
+        download_file_body["description"]
+        == f"Downloaded {download_filepath} to {local_item_path}."
+    )
 
-    # cfile_body_json = copy_file_res.body.decode('utf-8')
-    # cfile_body = json.loads(cfile_body_json)
-    # assert cfile_body["status"] == 'success'
-    # assert cfile_body['description'] == f'Copied {fs_root_path}/test_dir/file1.txt to /my_local_dir/file1.txt'
+    downloaded_filepath = local_item_path + "/file1.txt"
+    print("downloaded_filepath: ", downloaded_filepath)
+    assert local_fs.exists(downloaded_filepath)
+
+    # download dir [other to local]
+    download_dirpath = "/my_mem_dir/test_dir"
+    download_dir_payload = {
+        "item_path": download_dirpath,
+        "content": local_item_path,
+        "destination_key": local_key,
+        "action": "download",
+    }
+    download_dir_res = await jp_fetch(
+        "jupyter_fsspec",
+        "files",
+        "transfer",
+        method="POST",
+        params={"key": remote_key},
+        body=json.dumps(download_dir_payload),
+    )
 
-    # copy dir
+    download_dir_body_json = download_dir_res.body.decode("utf-8")
+    download_dir_body = json.loads(download_dir_body_json)
+    assert download_dir_body["status"] == "success"
+    assert (
+        download_dir_body["description"]
+        == f"Downloaded {download_dirpath} to {local_item_path}."
+    )
 
-    # move file
+    downloaded_dirpath = local_item_path + "/test_dir/"
+    print("downloaded_dirpath: ", downloaded_dirpath)
+    print("ls of downloaded_dirpath: ", local_fs.ls(local_item_path))
+    assert local_fs.exists(downloaded_dirpath)
 
-    # move dir
+    downloaded_dir_asset_path = local_item_path + "/test_dir/file1.txt"
+    print("downloaded_dir_asset_path: ", downloaded_dir_asset_path)
+    assert local_fs.exists(downloaded_dir_asset_path)

Diff for: src/handler/fileOperations.ts

@@ -89,12 +89,9 @@ export class FsspecModel {
   async refreshConfig() {
     // TODO fix/refactor
     this.userFilesystems = {};
-    Logger.debug('aaa');
     Logger.debug('[FSSpec] Refresh config requested');
     try {
-      Logger.debug('bbb');
       for (let i = 0; i < this.retry; i++) {
-        Logger.debug('ccc');
         Logger.info('[FSSpec] Attempting to read config file...');
         const result = await this.getStoredFilesystems(); // This is a result dict, not a response
         if (result?.status === 'success') {
@@ -107,16 +104,13 @@ export class FsspecModel {
           // Set active filesystem to first
           if (Object.keys(result).length > 0) {
             this.activeFilesystem = Object.keys(this.userFilesystems)[0];
-            Logger.debug('ddd');
           }
           break;
         } else {
-          Logger.debug('eee');
           // TODO handle no config file
           Logger.error('[FSSpec] Error fetching filesystems from user config');
           if (i + 1 < this.retry) {
             Logger.info('[FSSpec]   retrying...');
-            Logger.debug('fffr');
           }
         }
       }
@@ -125,7 +119,6 @@ export class FsspecModel {
         `[FSSpec] Error: Unknown error initializing fsspec model:\n${error}`
       );
     }
-    Logger.debug('zzz');
   }
 
   async getStoredFilesystems(): Promise<any> {
@@ -224,26 +217,6 @@ export class FsspecModel {
   }
 
   async delete(key: string, item_path: string): Promise<any> {
-    try {
-      const reqBody = JSON.stringify({
-        key,
-        item_path
-      });
-      const response = await requestAPI<any>('fsspec', {
-        method: 'DELETE',
-        body: reqBody,
-        headers: {
-          'Content-Type': 'application/json'
-        }
-      });
-      console.log('response is: ', response);
-    } catch (error) {
-      console.error('Failed to delete: ', error);
-      return null;
-    }
-  }
-
-  async delete_refactored(key: string, item_path: string): Promise<any> {
     try {
       const query = new URLSearchParams({
         key,
@@ -337,6 +310,62 @@ export class FsspecModel {
     }
   }
 
+  async upload(
+    key: string,
+    local_path: string,
+    remote_path: string,
+    action: 'upload'
+  ): Promise<any> {
+    try {
+      const query = new URLSearchParams({ action: action });
+      const reqBody = JSON.stringify({
+        key: key,
+        local_path,
+        remote_path
+      });
+
+      const response = await requestAPI<any>(
+        `files/transfer?${query.toString()}`,
+        {
+          method: 'POST',
+          body: reqBody,
+          headers: {
+            'Content-Type': 'application/json'
+          }
+        }
+      );
+      console.log('response: ', response);
+    } catch (error) {
+      console.error('Failed to upload: ', error);
+      return null;
+    }
+  }
+
+  async download(
+    key: string,
+    remote_path: string,
+    local_path: string,
+    action: 'download'
+  ): Promise<any> {
+    try {
+      const query = new URLSearchParams({ action: action });
+      const reqBody = JSON.stringify({
+        key: key,
+        remote_path,
+        local_path
+      });
+      await requestAPI<any>(`files/transfer?${query.toString()}`, {
+        method: 'POST',
+        body: reqBody,
+        headers: {
+          'Content-Type': 'application/json'
+        }
+      });
+    } catch (error) {
+      console.error('Failed to download: ', error);
+    }
+  }
+
   // async update(
   //   key: any = 'local%7CSourceDisk%7C.',
   //   item_path = '',