WIP: GEDS Daemon Mode.

CMakeLists.txt

@@ -20,7 +20,7 @@ option(HAVE_PROMETHEUS_HISTOGRAM_BUCKETS "Enable Prometheus Histogram Buckets."
 option(ENABLE_PROFILING "Compile with -pg to enable gprof." OFF)
 
 set(DEFAULT_BUCKET_NAME "default" CACHE STRING "Name of the default bucket.")
-set(BOOST_VERSION "1.80.0" CACHE STRING "Boost version")
+set(BOOST_VERSION "1.83.0" CACHE STRING "Boost version")
 
 message(STATUS "HAVE_TESTS:           ${HAVE_TESTS}")
 message(STATUS "HAVE_PYTHON_BINDINGS: ${HAVE_PYTHON_BINDINGS}")
@@ -147,6 +147,18 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(magic_enum)
 
+# # Capnproto
+FetchContent_Declare(
+    CapnProto
+    GIT_REPOSITORY https://github.com/capnproto/capnproto
+    GIT_TAG v1.0.1
+)
+set(CapnProto_CXXFLAGS "-Wno-deprecated")
+set(CapnProto_BUILD_TESTING OFF CACHE INTERNAL "")
+set(CapnProto_CAPNP_LITE ON CACHE INTERNAL "")
+FetchContent_MakeAvailable(CapnProto)
+
+
 # S3
 if(DEFINED ENV{AWS_SDK_INSTALL_DIR})
     set(AWSSDK_ROOT $ENV{AWS_SDK_INSTALL_DIR})

DEPENDENCIES

@@ -11,7 +11,7 @@ export AWS_SDK_VERSION="1.10.20"
 # GRPC Version
 export GRPC_VERSION="v1.51.1"
 # Boost Version
-export BOOST_VERSION="1.80.0"
+export BOOST_VERSION="1.83.0"
 
 # GEDS Docker Version
 export GEDS_DOCKER_VERSION="1.0"

doc/GEDS Daemon Mode.md

@@ -0,0 +1,146 @@
+# GEDS Daemon Mode Redesign.
+
+GEDS shall be redesigned to support multiple local instances. A central daemon is responsible
+for managing files and sharing files between other daemons.
+GEDS clients write and read to a mounted CSI hostpath which shares the data between pods.
+NVMeoF and GPU Direct RDMA should be supported in a second step.
+
+Goal:
+- Enable data-sharing of GEDS objects between multiple Python processes.
+- Enable data-sharing of GEDS objects across Kubernetes pods using a CSI hostpath.
+- Data should survive crashes.
+- Facilitate encryption.
+- Enable reading/writing data to/from GPU and/or NVMe directly
+- Native integration with Kubernetes
+
+
+## Overview
+
+The diagram shows a high-level overview.
+```
+ ┌────────────────────────────────────────────────────────────────────────────┐
+ │ Kubernetes Node                                                            │
+ │                                ┌────────────────────────────────────────┐  │
+ │                                │ Kubernetes Pod                         │  │
+ │                     IPC        │   ┌─────────────┐ ┌─────────────────┐  │  │
+ │                    ┌───────────┼───► GEDS Client │ │ Apache Spark    │  │  │
+ │                    │           │   └─────────────┘ └─────────────────┘  │  │
+ │                    │           │                                        │  │
+ │  ┌─────────────────┼────┐      └────────────────────────────────────────┘  │
+ │  │ Kubernetes Pod  │    │                                                  │
+ │  │  ┌──────────────▼──┐ │      ┌────────────────────────────────────────┐  │
+ │  │  │ GEDS Daemon     ◄─┼──┐   │ Kubernetes Pod                         │  │
+ │  │  └─────────────────┘ │  │   │   ┌─────────────┐ ┌──────────────────┐ │  │
+ │  │                      │  ├───┼───► GEDS Client │ │ Python + Pytorch │ │  │
+ │  │  ┌─────────────────┐ │  │   │   └─────────────┘ └──────────────────┘ │  │
+ │  │  │ PVC / Local SSD │ │  │   │                                        │  │
+ │  │  └─────────────────┘ │  │   │   ┌─────────────┐ ┌──────────────────┐ │  │
+ │  │                      │  └───┼───► GEDS Client │ │ Python + Pytorch │ │  │
+ │  └────────────────▲─────┘      │   └─────────────┘ └──────────────────┘ │  │
+ │                   │            │                                        │  │
+ │                   │            └────────────────────────────────────────┘  │
+ │                   │ TCP/IP                                                 │
+ │                   │                                                        │
+ └───────────────────┼────────────────────────────────────────────────────────┘
+                     │
+ ┌───────────────────┼────────────────────────────────────────────────────────┐
+ │ Kubernetes Node   │                                                        │
+ │                   │                                                        │
+ │  ┌────────────────▼─────┐      ┌────────────────────────────────────┐      │
+ │  │ Kubernetes Pod       │      │ Kubernetes Pod                     │      │
+ │  │  ┌─────────────────┐ │      │  ┌───────────────────────────────┐ │      │
+ │  │  │ GEDS Daemon     │ │      │  │ GEDS Client                   │ │      │
+ │  │  └─────────────────┘ │      │  └─────▲────────────────▲────▲───┘ │      │
+ │  │                      │      │        │ I/O + MMAP     │    │     │      │
+ │  │  ┌─────────────────┐ │Mounts│  ┌─────▼──────────┐     │    │     │      │
+ │  │  │ PVC / Local SSD ◄─┼──────┼──┤  Hostpath CSI  │     │    │     │      │
+ │  │  └─────────────────┘ │      │  └────────────────┘     │    │     │      │
+ │  │                      │      │                         │    │     │      │
+ │  └──────────────────────┘      └─────────────────────────┼────┼─────┘      │
+ │                                                          │    │            │
+ │                                               GPU Direct │    │NVMeOF      │
+ │                                ┌─────────────────────────▼┐   │            │
+ │                                │  GPU                     │   │            │
+ │                                └───────────────────▲──────┘   │            │
+ │                                                    │          │            │
+ └────────────────────────────────────────────────────┼──────────┼────────────┘
+                                                      │          │
+                                      GPU Direct RDMA │          │
+ ┌────────────────────────────────────────────────────▼──────────▼────────────┐
+ │ NVMe JBOD                                                                  │
+ │                                                                            │
+ │                                                                            │
+ │                                                                            │
+ │                                                                            │
+ │                                                                            │
+ └────────────────────────────────────────────────────────────────────────────┘
+```
+
+## Operations
+
+### Creating a file
+
+1. GEDS Creates file with `BUCKET/PATH` and `UUID`.
+2. GEDS Client creates a temporary file in the hostpath volume.
+3. GEDS Client writes to the file.
+4. GEDS Client seals the file. This transfers the ownership of the file to the
+   GEDS daemon. FD is kept open and the daemon does a rename.
+5. GEDS Daemon tells the metadata server that a new file has been
+   created. 
+
+   Conflict resolution:
+    1. If no file with `BUCKET/PATH` does exist, the entry is created.
+    2. If `BUCKET/PATH` already exists, then the existing entry is overridden.
+
+6. GEDS Client reopens the file `Read-Only` and opens a file-descriptor.
+7. Other GEDS Clients are now able to open this file.
+
+### Opening a file
+
+1. GEDS Client tells the daemon that it wants to open `FILE`.
+2. GEDS Daemon:
+    1. Checks with Metadata Server if the `BUCKET/PATH` exists. Receives `UUID` and locations.
+    2. Increases the usage-count count of the filehandle to prevent deletion.
+    3. Tells the `PATH` and size of `FILE` to the `Client`.
+
+### Reading from a file
+
+A file can be on one of the following locations:
+
+- **Local**, if the file was created by a client on the same node, or if it is cached.
+- **DFS**, if the file was stored on a distributed file system.
+- **S3**, if the file is stored in a bucket on a S3-like service.
+- **Remote**, if the file is stored on a different GEDS Daemon.
+
+The GEDS Client will forward all reads to the GEDS Dameon for **Remote** files. All other file-types can be read directly by the GEDS Client.
+
+
+### Closing a file
+
+1. GEDS Client tells the daemon that it closed the `BUCKET/PATH`.
+2. GEDS Daemon:
+    1. Decreases the file-counter.
+    2. If the file-counter is zero and the file is marked as deleted, delete the file.
+
+### Deleting a file
+
+1. GEDS Client tells the daemon that it wants to delete `BUCKET/PATH`.
+2. GEDS Client closes local FD.
+2. GEDS Daemon:
+    1. Decreases reference counter for open `UUID`.
+    2. Tells the metadata server that the path `BUCKET/PATH` has been deleted.
+    3. Marks file as deleted. Delete the file if the reference counter is zero.
+
+### Garbage collection
+
+1. A daemon process automatically checks local file-usage.
+2. If the defined quota is reached no files can be created.
+3. Local copies can be deleted if:
+    1. The file is marked as deleted.
+    2. The file is not open and other copies exist (UUID matches).
+    3. A previous version of the file is not referenced any more.
+
+## Technical Implementations
+
+- The TCP/IP transport is a GEDS Client as well. This way the transport requires no extra
+  file/open/close mechanisms.

src/CMakeLists.txt

@@ -14,6 +14,7 @@ add_subdirectory(s3)
 add_subdirectory(libgeds)
 
 # Binaries
+add_subdirectory(daemon)
 add_subdirectory(benchmarks)
 add_subdirectory(geds)
 add_subdirectory(geds_cmd)

src/common/CMakeLists.txt

@@ -7,6 +7,8 @@
 set(SOURCES
     DirectoryMarker.cpp
     DirectoryMarker.h
+    Filesystem.cpp
+    Filesystem.h
     GEDSFileStatus.h
 )
 
@@ -18,6 +20,12 @@ target_compile_definitions(geds_common
         "HAVE_RDMA=$<BOOL:${HAVE_RDMA}>"
         _POSIX_C_SOURCE=200809L
 )
+
+target_link_libraries(
+        geds_common PUBLIC 
+        absl::status
+)
+
 target_include_directories(geds_common
         PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}

src/libgeds/CMakeLists.txt

@@ -4,55 +4,14 @@
 #
 
 SET(SOURCES
-        Filesystem.cpp
-        Filesystem.h
-        FileTransferProtocol.cpp
-        FileTransferProtocol.h
-        FileTransferService.cpp
-        FileTransferService.h
-
-        TcpTransport.h
-        TcpTransport.cpp
-
         GEDS.cpp
         GEDS.h
-        GEDSAbstractFileHandle.cpp
-        GEDSAbstractFileHandle.h
-        GEDSCachedFileHandle.cpp
-        GEDSCachedFileHandle.h
-        GEDSConfig.h
-        GEDSConfig.cpp
         GEDSFile.h
         GEDSFile.cpp
         GEDSFileHandle.h
         GEDSFileHandle.cpp
-        GEDSProtocol.h
-        GEDSProtocol.cpp
-        GEDSRelocatableFileHandle.cpp
-        GEDSRelocatableFileHandle.h
-        GEDSService.cpp
-        GEDSService.h
-        HttpServer.cpp
-        HttpServer.h
-        HttpSession.cpp
-        HttpSession.h
-        MetadataService.cpp
-        MetadataService.h
-        GEDSInternal.cpp
-        GEDSInternal.h
-        GEDSLocalFileHandle.h
-        GEDSMMapFileHandle.h
         GEDSRemoteFileHandle.cpp
         GEDSRemoteFileHandle.h
-        GEDSS3FileHandle.cpp
-        GEDSS3FileHandle.h
-
-        LocalFile.cpp
-        LocalFile.h
-        MMAPFile.cpp
-        MMAPFile.h
-        Server.cpp
-        Server.h
 )
 
 # Create an object lib to build both a dynamic and a static library.
@@ -107,6 +66,6 @@ target_link_libraries(libgeds_dynamic PUBLIC geds_objlib)
 install(TARGETS libgeds libgeds_dynamic
         COMPONENT geds)
 
-if(HAVE_TESTS)
-        add_subdirectory(tests)
-endif()
+# if(HAVE_TESTS)
+#         add_subdirectory(tests)
+# endif()