File system adapters (#204)

* file_system_adapters: update environment

* file_system_adapters: new adapters for local storage access

* file_system_adapters: use new models and adapters

* file_system_adapters: update tests

* file_system_adapters: add gitkeep to test directory

* Update job_executor/adapter/fs/datastore_files.py

Co-authored-by: pawbu <pawbu@users.noreply.github.com>

---------

Co-authored-by: pawbu <pawbu@users.noreply.github.com>

Author: DanielElisenberg

.test.env

@@ -1,7 +1,4 @@
-INPUT_DIR=tests/resources/datastores/TEST_DATASTORE_input
-WORKING_DIR=tests/resources/datastores/TEST_DATASTORE_working
 DATASTORE_DIR=tests/resources/datastores/TEST_DATASTORE
-RSA_KEYS_DIRECTORY=tests/resources/rsa_keys
 PSEUDONYM_SERVICE_URL=http://mock.pseudonym.service
 DATASTORE_API_URL=http://mock.job.service
 NUMBER_OF_WORKERS=4

job_executor/adapter/datastore_api/models.py

@@ -3,7 +3,7 @@
 
 from pydantic import model_validator
 
-from job_executor.adapter.local_storage.models.datastore_versions import (
+from job_executor.adapter.fs.models.datastore_versions import (
     DatastoreVersion,
 )
 from job_executor.common.models import CamelModel

job_executor/adapter/fs/__init__.py

@@ -0,0 +1,49 @@
+import shutil
+from pathlib import Path
+from typing import Protocol
+
+from job_executor.adapter.fs.datastore_files import DatastoreDirectory
+from job_executor.adapter.fs.input_files import InputDirectory
+from job_executor.adapter.fs.working_files import WorkingDirectory
+
+
+class FileSystemAdapter(Protocol):
+    datastore_dir: DatastoreDirectory
+    working_dir: WorkingDirectory
+    input_dir: InputDirectory
+
+    def move_working_dir_parquet_to_datastore(
+        self, dataset_name: str
+    ) -> None: ...
+
+
+class LocalStorageAdapter:
+    datastore_dir: DatastoreDirectory
+    working_dir: WorkingDirectory
+    input_dir: InputDirectory
+
+    def __init__(self, datastore_dir_path: Path) -> None:
+        self.datastore_dir = DatastoreDirectory(datastore_dir_path)
+        self.working_dir = WorkingDirectory(
+            Path(f"{datastore_dir_path}_working")
+        )
+        self.input_dir = InputDirectory(Path(f"{datastore_dir_path}_input"))
+
+    def move_working_dir_parquet_to_datastore(self, dataset_name: str) -> None:
+        """
+        Moves the given parquet DRAFT file from the working directory to
+        the appropriate datastore sub directory.
+
+        * dataset_name: str - name of dataset
+        """
+        working_dir_parquet_path = self.working_dir._get_draft_parquet_path(
+            dataset_name
+        )
+        shutil.move(
+            working_dir_parquet_path,
+            (
+                self.datastore_dir.data_dir
+                / dataset_name
+                / working_dir_parquet_path.parts[-1]
+            ),
+        )