[LibOS] Single-process-lifetime rollback protection for protected files (WIP)

* adds libos_encrypted_volume as mount-data for protected fileystem which
  includes map <name, <last-root-hash, ...>> to keep track of root hashes
  across open/close cycles of a particular value, ensuring consistency across
  the whole enclave life-time

Signed-off-by: g2flyer <[email protected]>

Author: g2flyer

common/src/protected_files/protected_files.c

@@ -1125,7 +1125,7 @@ static void ipf_delete_cache(pf_context_t* pf) {
     }
 }
 
-static bool ipf_close(pf_context_t* pf) {
+static bool ipf_close(pf_context_t* pf, pf_mac_t* closing_root_gmac) {
     bool retval = true;
 
     if (pf->file_status != PF_STATUS_SUCCESS) {
@@ -1138,6 +1138,10 @@ static bool ipf_close(pf_context_t* pf) {
         }
     }
 
+    if (closing_root_gmac != NULL) {
+        memcpy(*closing_root_gmac, pf->file_metadata.plain_part.metadata_gmac, sizeof(pf_mac_t));
+    }
+
     // omeg: fs close is done by Gramine handler
     pf->file_status = PF_STATUS_UNINITIALIZED;
 
@@ -1176,20 +1180,25 @@ void pf_set_callbacks(pf_read_f read_f, pf_write_f write_f, pf_fsync_f fsync_f,
 }
 
 pf_status_t pf_open(pf_handle_t handle, const char* path, uint64_t underlying_size,
-                    pf_file_mode_t mode, bool create, const pf_key_t* key, pf_context_t** context) {
+                    pf_file_mode_t mode, bool create, const pf_key_t* key,
+                    pf_mac_t* opening_root_gmac, pf_context_t** context) {
     if (!g_initialized)
         return PF_STATUS_UNINITIALIZED;
 
     pf_status_t status;
     *context = ipf_open(path, mode, create, handle, underlying_size, key, &status);
+    if (opening_root_gmac != NULL) {
+        memcpy(*opening_root_gmac, (*context)->file_metadata.plain_part.metadata_gmac,
+               sizeof(pf_mac_t));
+    }
     return status;
 }
 
-pf_status_t pf_close(pf_context_t* pf) {
+pf_status_t pf_close(pf_context_t* pf, pf_mac_t* closing_root_gmac) {
     if (!g_initialized)
         return PF_STATUS_UNINITIALIZED;
 
-    if (ipf_close(pf)) {
+    if (ipf_close(pf, closing_root_gmac)) {
         free(pf);
         return PF_STATUS_SUCCESS;
     }
@@ -1259,7 +1268,7 @@ pf_status_t pf_set_size(pf_context_t* pf, uint64_t size) {
     return PF_STATUS_SUCCESS;
 }
 
-pf_status_t pf_rename(pf_context_t* pf, const char* new_path) {
+pf_status_t pf_rename(pf_context_t* pf, const char* new_path, pf_mac_t* new_root_gmac) {
     if (!g_initialized)
         return PF_STATUS_UNINITIALIZED;
 
@@ -1275,6 +1284,9 @@ pf_status_t pf_rename(pf_context_t* pf, const char* new_path) {
     pf->need_writing = true;
     if (!ipf_internal_flush(pf))
         return pf->last_error;
+    if (new_root_gmac != NULL) {
+        memcpy(*new_root_gmac, pf->file_metadata.plain_part.metadata_gmac, sizeof(pf_mac_t));
+    }
 
     return PF_STATUS_SUCCESS;
 }
@@ -1348,3 +1360,7 @@ pf_status_t pf_flush(pf_context_t* pf) {
 
     return PF_STATUS_SUCCESS;
 }
+
+void pf_set_corrupted(pf_context_t* pf) {
+    pf->file_status = PF_STATUS_CORRUPTED;
+}

common/src/protected_files/protected_files.h

@@ -26,6 +26,11 @@ typedef uint8_t pf_mac_t[PF_MAC_SIZE];
 typedef uint8_t pf_key_t[PF_KEY_SIZE];
 typedef uint8_t pf_keyid_t[32]; /* key derivation material */
 
+// convenience macros to print out some mac fingerprint: printf( "some text " MAC_PRINTF_PATTERN "
+// yet other text", MAC_PRINTF_ARGS(mac) );
+#define MAC_PRINTF_PATTERN   "0x%02x%02x%02x%02x..."
+#define MAC_PRINTF_ARGS(mac) (mac)[0], (mac)[1], (mac)[2], (mac)[3]
+
 typedef enum _pf_status_t {
     PF_STATUS_SUCCESS              = 0,
     PF_STATUS_UNKNOWN_ERROR        = -1,
@@ -209,28 +214,31 @@ const char* pf_strerror(int err);
 /*!
  * \brief Open a protected file.
  *
- * \param      handle           Open underlying file handle.
- * \param      path             Path to the file. If NULL and \p create is false, don't check path
- *                              for validity.
- * \param      underlying_size  Underlying file size.
- * \param      mode             Access mode.
- * \param      create           Overwrite file contents if true.
- * \param      key              Wrap key.
- * \param[out] context          PF context for later calls.
+ * \param      handle             Open underlying file handle.
+ * \param      path               Path to the file. If NULL and \p create is false, don't check path
+ *                                for validity.
+ * \param      underlying_size    Underlying file size.
+ * \param      mode               Access mode.
+ * \param      create             Overwrite file contents if true.
+ * \param      key                Wrap key.
+ * \param      opening_root_gmac  If non-NULL, !create & successfull open, returns root-hash of file
+ * \param[out] context            PF context for later calls.
  *
  * \returns PF status.
  */
 pf_status_t pf_open(pf_handle_t handle, const char* path, uint64_t underlying_size,
-                    pf_file_mode_t mode, bool create, const pf_key_t* key, pf_context_t** context);
+                    pf_file_mode_t mode, bool create, const pf_key_t* key,
+                    pf_mac_t* opening_root_gmac, pf_context_t** context);
 
 /*!
  * \brief Close a protected file and commit all changes to disk.
  *
- * \param pf  PF context.
+ * \param pf                 PF context.
+ * \param closing_root_gmac  If non-NULL, returns root-hash of file at closing time
  *
  * \returns PF status.
  */
-pf_status_t pf_close(pf_context_t* pf);
+pf_status_t pf_close(pf_context_t* pf, pf_mac_t* closing_root_gmac);
 
 /*!
  * \brief Read from a protected file.
@@ -283,13 +291,14 @@ pf_status_t pf_set_size(pf_context_t* pf, uint64_t size);
 /*!
  * \brief Rename a PF.
  *
- * \param pf        PF context.
- * \param new_path  New file path.
+ * \param pf             PF context.
+ * \param new_path       New file path.
+ * \param new_root_gmac  if non-NULL, returns new root-hash of file
  *
  * Updates the path inside protected file header, and flushes all changes. The caller is responsible
  * for renaming the underlying file.
  */
-pf_status_t pf_rename(pf_context_t* pf, const char* new_path);
+pf_status_t pf_rename(pf_context_t* pf, const char* new_path, pf_mac_t* new_root_gmac);
 
 /*!
  * \brief Flush any pending data of a protected file to disk.
@@ -299,3 +308,10 @@ pf_status_t pf_rename(pf_context_t* pf, const char* new_path);
  * \returns PF status.
  */
 pf_status_t pf_flush(pf_context_t* pf);
+
+/*!
+ * \brief Set protected file state as corrupted
+ *
+ * \param pf  PF context.
+ */
+void pf_set_corrupted(pf_context_t* pf);

libos/include/libos_fs.h

@@ -35,6 +35,9 @@ struct libos_mount_params {
 
     /* Key name (used by `chroot_encrypted` filesystem), or NULL if not applicable */
     const char* key_name;
+
+    /* Enforcement type (used by `chroot_encrypted` filesystem), or NULL if not applicable */
+    const char* protection_mode;
 };
 
 struct libos_fs_ops {

libos/include/libos_fs_encrypted.h

@@ -14,6 +14,7 @@
 
 #include <stddef.h>
 
+#include "libos_checkpoint.h"  // for include of uthash.h _and_ consistent uthash_fatal macros
 #include "libos_types.h"
 #include "list.h"
 #include "pal.h"
@@ -34,6 +35,40 @@ struct libos_encrypted_files_key {
     LIST_TYPE(libos_encrypted_files_key) list;
 };
 
+typedef enum {
+    PF_FILE_IN_USE_NEW      = 0,  // file is currently in-use but did not exist at open time
+    PF_FILE_IN_USE_EXISTING = 1,  // file is currently in-use and existed at open time
+    PF_FILE_CLOSED          = 2,  // file was provisously seend with known (good committed) state
+    PF_FILE_DELETED         = 3,  // the old path of renames is also considered deleted
+    PF_FILE_ERROR           = 4,  // file is in non-determined state due to some errors
+} libos_encrypted_file_state_t;
+
+/*
+ * Map mapping file URIs to state providing information on files, in particular whether we have seen
+ * them before and what the last seen root-hash is.  This is necessary to provide rollback
+ */
+struct libos_encrypted_volume_state_map {
+    char* norm_path;  // assumptions: all paths canonicalized, symlinks are resolved & no hard links
+    libos_encrypted_file_state_t state;
+    pf_mac_t last_seen_root_gmac;
+    UT_hash_handle hh;
+};
+
+typedef enum {
+    PF_ENCLAVE_LIFE_RB_PROTECTION_NONE       = 0,
+    PF_ENCLAVE_LIFE_RB_PROTECTION_NON_STRICT = 1,
+    PF_ENCLAVE_LIFE_RB_PROTECTION_STRICT     = 2,
+} libos_encrypted_files_mode_t;
+
+struct libos_encrypted_volume {
+    libos_encrypted_files_mode_t protection_mode;
+
+    struct libos_encrypted_volume_state_map* files_state_map;
+    struct libos_lock files_state_map_lock;
+
+    struct libos_encrypted_files_key* key;
+};
+
 /*
  * Represents a specific encrypted file. The file is open as long as `use_count` is greater than 0.
  * Note that the file can be open and closed multiple times before it's destroyed.
@@ -44,7 +79,8 @@ struct libos_encrypted_files_key {
 struct libos_encrypted_file {
     size_t use_count;
     char* uri;
-    struct libos_encrypted_files_key* key;
+    char* norm_path;  // normalized path of of uri
+    struct libos_encrypted_volume* volume;
 
     /* `pf` and `pal_handle` are non-null as long as `use_count` is greater than 0 */
     pf_context_t* pf;
@@ -110,29 +146,29 @@ void update_encrypted_files_key(struct libos_encrypted_files_key* key, const pf_
  * \brief Open an existing encrypted file.
  *
  * \param      uri      PAL URI to open, has to begin with "file:".
- * \param      key      Key, has to be already set.
+ * \param      volume   Volume assocated with file, has to be already set.
  * \param[out] out_enc  On success, set to a newly created `libos_encrypted_file` object.
  *
  * `uri` has to correspond to an existing file that can be decrypted with `key`.
  *
  * The newly created `libos_encrypted_file` object will have `use_count` set to 1.
  */
-int encrypted_file_open(const char* uri, struct libos_encrypted_files_key* key,
+int encrypted_file_open(const char* uri, struct libos_encrypted_volume* volume,
                         struct libos_encrypted_file** out_enc);
 
 /*
  * \brief Create a new encrypted file.
  *
  * \param      uri      PAL URI to open, has to begin with "file:".
  * \param      perm     Permissions for the new file.
- * \param      key      Key, has to be already set.
+ * \param      volume   Volume assocated with file, has to be already set.
  * \param[out] out_enc  On success, set to a newly created `libos_encrypted_file` object.
  *
  * `uri` must not correspond to an existing file.
  *
  * The newly created `libos_encrypted_file` object will have `use_count` set to 1.
  */
-int encrypted_file_create(const char* uri, mode_t perm, struct libos_encrypted_files_key* key,
+int encrypted_file_create(const char* uri, mode_t perm, struct libos_encrypted_volume* volume,
                           struct libos_encrypted_file** out_enc);
 
 /*
@@ -154,7 +190,7 @@ int encrypted_file_get(struct libos_encrypted_file* enc);
  *
  * This decreases `use_count`, and closes the file if it reaches 0.
  */
-void encrypted_file_put(struct libos_encrypted_file* enc);
+void encrypted_file_put(struct libos_encrypted_file* enc, bool fs_reachable);
 
 /*
  * \brief Flush pending writes to an encrypted file.
@@ -166,6 +202,7 @@ int encrypted_file_read(struct libos_encrypted_file* enc, void* buf, size_t buf_
 int encrypted_file_write(struct libos_encrypted_file* enc, const void* buf, size_t buf_size,
                          file_off_t offset, size_t* out_count);
 int encrypted_file_rename(struct libos_encrypted_file* enc, const char* new_uri);
+int encrypted_file_unlink(struct libos_encrypted_file* enc);
 
 int encrypted_file_get_size(struct libos_encrypted_file* enc, file_off_t* out_size);
 int encrypted_file_set_size(struct libos_encrypted_file* enc, file_off_t size);