Fixed data corruption with multiple write handles

Multiple write handles in littlefs has always been a bit confusing
(hopefully improving in littlefs3), but I didn't realize it could lead
to corrupted data.

The problem, as noted by Ictogan1, is that syncs to related file handles
ignores the LFS_F_DIRTY flag. If you open a file twice, and write to one
handle, littlefs doesn't realize the other handle it out-of-date.

This may not seem like a problem, but then littlefs is happy to
reallocate those (still referenced) blocks, leading to data corruption:

  open(a, "quiche.txt")
  write(a)
  sync(a) // syncs a's contents
  open(b, "quiche.txt")
  truncate(b)
  sync(b) // syncs b's contents
  write(b) // may allocate from a
  rewind(a)
  read(a) // potentially corrupted

---

What we want is to set LFS_F_DIRTY in all other file handles during
lfs_file_sync, but doing so would force those file handles to sync
during close. That would be even more confusing (not to mention
backwards incompatible).

In theory setting both LFS_F_DIRTY + LFS_F_ERRED could work, but that
would prevent implicit syncs of writes that haven't actually errored:

  open(a, "quiche.txt")
  write(a)
  open(b, "quiche.txt")
  write(b)
  close(b) // syncs b's contents
  close(a) // should sync a's contents

So, instead, as a somewhat clunky workaround, a new flag: LFS_F_DUSTY,
which indicates a file does not match storage, but should not be synced
during close.

---

It's worth noting this is already fixed in littlefs3, which includes a
more rigorous, and hopefully easier to use sync model. But in the
meantime, this should at least prevent the loss of data.

Added test_alloc_multihandle and test_alloc_multihandle_reuse to prevent
a regression, test_alloc_multihandle_reuse does reproduce the bug.

Found and reproduced by Ictogan1

Author: geky

lfs.c

@@ -3244,10 +3244,12 @@ static int lfs_file_open_(lfs_t *lfs, lfs_file_t *file,
 #endif
 
 static int lfs_file_close_(lfs_t *lfs, lfs_file_t *file) {
-#ifndef LFS_READONLY
-    int err = lfs_file_sync_(lfs, file);
-#else
     int err = 0;
+#ifndef LFS_READONLY
+    // it's not safe to do anything if our file errored
+    if (!(file->flags & LFS_F_ERRED)) {
+        err = lfs_file_sync_(lfs, file);
+    }
 #endif
 
     // remove from list of mdirs
@@ -3429,18 +3431,12 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
 
 #ifndef LFS_READONLY
 static int lfs_file_sync_(lfs_t *lfs, lfs_file_t *file) {
-    if (file->flags & LFS_F_ERRED) {
-        // it's not safe to do anything if our file errored
-        return 0;
-    }
-
     int err = lfs_file_flush(lfs, file);
     if (err) {
         file->flags |= LFS_F_ERRED;
         return err;
     }
 
-
     if ((file->flags & LFS_F_DIRTY) &&
             !lfs_pair_isnull(file->m.pair)) {
         // before we commit metadata, we need sync the disk to make sure
@@ -3485,6 +3481,17 @@ static int lfs_file_sync_(lfs_t *lfs, lfs_file_t *file) {
         file->flags &= ~LFS_F_DIRTY;
     }
 
+    // mark any other file handles as dirty + desync
+    for (lfs_file_t *f = (lfs_file_t*)lfs->mlist; f; f = f->next) {
+        if (file != f
+                && f->type == LFS_TYPE_REG
+                && lfs_pair_cmp(f->m.pair, file->m.pair) == 0
+                && f->id == file->id) {
+            f->flags |= LFS_F_DUSTY;
+        }
+    }
+
+    file->flags &= ~LFS_F_ERRED & ~LFS_F_DUSTY;
     return 0;
 }
 #endif
@@ -3692,7 +3699,7 @@ static lfs_ssize_t lfs_file_write_(lfs_t *lfs, lfs_file_t *file,
         return nsize;
     }
 
-    file->flags &= ~LFS_F_ERRED;
+    file->flags &= ~LFS_F_ERRED & ~LFS_F_DUSTY;
     return nsize;
 }
 #endif
@@ -4771,7 +4778,8 @@ int lfs_fs_traverse_(lfs_t *lfs,
             continue;
         }
 
-        if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) {
+        if (((f->flags & LFS_F_DIRTY) || (f->flags & LFS_F_DUSTY))
+                && !(f->flags & LFS_F_INLINE)) {
             int err = lfs_ctz_traverse(lfs, &f->cache, &lfs->rcache,
                     f->ctz.head, f->ctz.size, cb, data);
             if (err) {

lfs.h

@@ -135,14 +135,15 @@ enum lfs_open_flags {
 
     // internally used flags
 #ifndef LFS_READONLY
-    LFS_F_DIRTY   = 0x010000, // File does not match storage
-    LFS_F_WRITING = 0x020000, // File has been written since last flush
+    LFS_F_DIRTY   = 0x00010000, // File does not match storage due to write
+    LFS_F_DUSTY   = 0x00020000, // File does not match storage due to desync
+    LFS_F_WRITING = 0x00040000, // File has been written since last flush
 #endif
-    LFS_F_READING = 0x040000, // File has been read since last flush
+    LFS_F_READING = 0x00080000, // File has been read since last flush
 #ifndef LFS_READONLY
-    LFS_F_ERRED   = 0x080000, // An error occurred during write
+    LFS_F_ERRED   = 0x00100000, // An error occurred during write
 #endif
-    LFS_F_INLINE  = 0x100000, // Currently inlined in directory entry
+    LFS_F_INLINE  = 0x01000000, // Currently inlined in directory entry
 };
 
 // File seek flags

tests/test_alloc.toml

@@ -250,6 +250,189 @@ code = '''
     }
 '''
 
+# multiple handle allocation test
+#
+# this tests that multiple open handles to the same file don't clobber
+# each other
+[cases.test_alloc_multihandle]
+defines.FILES = 2
+defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)'
+defines.GC = [false, true]
+defines.COMPACT_THRESH = ['-1', '0', 'BLOCK_SIZE/2']
+defines.INFER_BC = [false, true]
+defines.SYNC = [false, true]
+code = '''
+    const char *names[] = {"eggs", "spinach"};
+    lfs_file_t files[FILES];
+
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    struct lfs_config cfg_ = *cfg;
+    if (INFER_BC) {
+        cfg_.block_count = 0;
+    }
+    lfs_mount(&lfs, &cfg_) => 0;
+    lfs_mkdir(&lfs, "breakfast") => 0;
+
+    // write one file
+    char path[1024];
+    sprintf(path, "breakfast/quiche");
+    lfs_file_open(&lfs, &files[0], path,
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    if (GC) {
+        lfs_fs_gc(&lfs) => 0;
+    }
+    size_t size = strlen(names[0]);
+    for (lfs_size_t i = 0; i < SIZE; i += size) {
+        lfs_file_write(&lfs, &files[0], names[0], size) => size;
+    }
+    // sync?
+    if (SYNC) {
+        lfs_file_sync(&lfs, &files[0]) => 0;
+    }
+
+    // write the other file
+    sprintf(path, "breakfast/quiche");
+    lfs_file_open(&lfs, &files[1], path,
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_TRUNC) => 0;
+    if (GC) {
+        lfs_fs_gc(&lfs) => 0;
+    }
+    size = strlen(names[1]);
+    for (lfs_size_t i = 0; i < SIZE; i += size) {
+        lfs_file_write(&lfs, &files[1], names[1], size) => size;
+    }
+    // sync?
+    if (SYNC) {
+        lfs_file_sync(&lfs, &files[1]) => 0;
+    }
+
+    // try to read from both
+    for (int n = 0; n < FILES; n++) {
+        lfs_file_rewind(&lfs, &files[n]) => 0;
+        size_t size = strlen(names[n]);
+        for (lfs_size_t i = 0; i < SIZE; i += size) {
+            uint8_t buffer[1024];
+            lfs_file_read(&lfs, &files[n], buffer, size) => size;
+            assert(memcmp(buffer, names[n], size) == 0);
+        }
+    }
+    for (int n = 0; n < FILES; n++) {
+        lfs_file_close(&lfs, &files[n]) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+
+    // check after remounting
+    lfs_mount(&lfs, &cfg_) => 0;
+    {
+        // last one wins
+        int n = FILES-1;
+        char path[1024];
+        sprintf(path, "breakfast/quiche");
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0;
+        size_t size = strlen(names[n]);
+        for (lfs_size_t i = 0; i < SIZE; i += size) {
+            uint8_t buffer[1024];
+            lfs_file_read(&lfs, &file, buffer, size) => size;
+            assert(memcmp(buffer, names[n], size) == 0);
+        }
+        lfs_file_close(&lfs, &file) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+# multiple handle allocation reuse test
+[cases.test_alloc_multihandle_reuse]
+defines.FILES = 2
+defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / (FILES+1))'
+defines.CYCLES = [1, 10]
+defines.INFER_BC = [false, true]
+defines.SYNC = [false, true]
+code = '''
+    const char *names[] = {"eggs", "spinach"};
+    lfs_file_t files[FILES];
+
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    struct lfs_config cfg_ = *cfg;
+    if (INFER_BC) {
+        cfg_.block_count = 0;
+    }
+
+    lfs_mount(&lfs, &cfg_) => 0;
+    lfs_mkdir(&lfs, "breakfast") => 0;
+
+    // write one file
+    char path[1024];
+    sprintf(path, "breakfast/quiche");
+    lfs_file_open(&lfs, &files[0], path,
+            LFS_O_RDWR | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    if (GC) {
+        lfs_fs_gc(&lfs) => 0;
+    }
+    size_t size = strlen(names[0]);
+    for (lfs_size_t i = 0; i < SIZE; i += size) {
+        lfs_file_write(&lfs, &files[0], names[0], size) => size;
+    }
+    // sync?
+    if (SYNC) {
+        lfs_file_sync(&lfs, &files[0]) => 0;
+    }
+
+    for (int c = 0; c < CYCLES; c++) {
+        // write the other file
+        sprintf(path, "breakfast/quiche");
+        lfs_file_open(&lfs, &files[1], path,
+                LFS_O_RDWR | LFS_O_CREAT | LFS_O_TRUNC) => 0;
+        if (GC) {
+            lfs_fs_gc(&lfs) => 0;
+        }
+        size = strlen(names[1]);
+        for (lfs_size_t i = 0; i < SIZE; i += size) {
+            lfs_file_write(&lfs, &files[1], names[1], size) => size;
+        }
+        // sync?
+        if (SYNC) {
+            lfs_file_sync(&lfs, &files[1]) => 0;
+        }
+
+        // try to read from both
+        for (int n = 0; n < FILES; n++) {
+            lfs_file_rewind(&lfs, &files[n]) => 0;
+            size_t size = strlen(names[n]);
+            for (lfs_size_t i = 0; i < SIZE; i += size) {
+                uint8_t buffer[1024];
+                lfs_file_read(&lfs, &files[n], buffer, size) => size;
+                assert(memcmp(buffer, names[n], size) == 0);
+            }
+        }
+
+        lfs_file_close(&lfs, &files[1]) => 0;
+    }
+    lfs_file_close(&lfs, &files[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // check after remounting
+    lfs_mount(&lfs, &cfg_) => 0;
+    {
+        // last one wins
+        int n = (SYNC) ? FILES-1 : 0;
+        char path[1024];
+        sprintf(path, "breakfast/quiche");
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, path, LFS_O_RDONLY) => 0;
+        size_t size = strlen(names[n]);
+        for (int i = 0; i < SIZE; i += size) {
+            uint8_t buffer[1024];
+            lfs_file_read(&lfs, &file, buffer, size) => size;
+            assert(memcmp(buffer, names[n], size) == 0);
+        }
+        lfs_file_close(&lfs, &file) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
 # exhaustion test
 [cases.test_alloc_exhaustion]
 defines.INFER_BC = [false, true]