Skip to content

Commit 8c458fa

Browse files
gekygeky
authored
Merge pull request #1094 from sosthene-nitrokey/shrink-fs
Add support for shrinking a filesystem
2 parents 3149201 + edaaaf8 commit 8c458fa

5 files changed

Lines changed: 285 additions & 24 deletions

File tree

‎.github/workflows/test.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,22 @@ jobs:
374374
run: |
375375
CFLAGS="$CFLAGS -DLFS_NO_INTRINSICS" make test
376376
377+
test-shrink:
378+
runs-on: ubuntu-latest
379+
steps:
380+
- uses: actions/checkout@v4
381+
- name: install
382+
run: |
383+
# need a few things
384+
sudo apt-get update -qq
385+
sudo apt-get install -qq gcc python3 python3-pip
386+
pip3 install toml
387+
gcc --version
388+
python3 --version
389+
- name: test-no-intrinsics
390+
run: |
391+
CFLAGS="$CFLAGS -DLFS_SHRINKNONRELOCATING" make test
392+
377393
# run with all trace options enabled to at least make sure these
378394
# all compile
379395
test-yes-trace:

‎lfs.c‎

Lines changed: 47 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5237,40 +5237,64 @@ static int lfs_fs_gc_(lfs_t *lfs) {
52375237
#endif
52385238

52395239
#ifndef LFS_READONLY
5240+
#ifdef LFS_SHRINKNONRELOCATING
5241+
static int lfs_shrink_checkblock(void *data, lfs_block_t block) {
5242+
lfs_size_t threshold = *((lfs_size_t*)data);
5243+
if (block >= threshold) {
5244+
return LFS_ERR_NOTEMPTY;
5245+
}
5246+
return 0;
5247+
}
5248+
#endif
5249+
52405250
static int lfs_fs_grow_(lfs_t *lfs, lfs_size_t block_count) {
5241-
// shrinking is not supported
5242-
LFS_ASSERT(block_count >= lfs->block_count);
5251+
int err;
52435252

5244-
if (block_count > lfs->block_count) {
5245-
lfs->block_count = block_count;
5253+
if (block_count == lfs->block_count) {
5254+
return 0;
5255+
}
52465256

5247-
// fetch the root
5248-
lfs_mdir_t root;
5249-
int err = lfs_dir_fetch(lfs, &root, lfs->root);
5257+
5258+
#ifndef LFS_SHRINKNONRELOCATING
5259+
// shrinking is not supported
5260+
LFS_ASSERT(block_count >= lfs->block_count);
5261+
#endif
5262+
#ifdef LFS_SHRINKNONRELOCATING
5263+
if (block_count < lfs->block_count) {
5264+
err = lfs_fs_traverse_(lfs, lfs_shrink_checkblock, &block_count, true);
52505265
if (err) {
52515266
return err;
52525267
}
5268+
}
5269+
#endif
52535270

5254-
// update the superblock
5255-
lfs_superblock_t superblock;
5256-
lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0),
5257-
LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
5258-
&superblock);
5259-
if (tag < 0) {
5260-
return tag;
5261-
}
5262-
lfs_superblock_fromle32(&superblock);
5271+
lfs->block_count = block_count;
52635272

5264-
superblock.block_count = lfs->block_count;
5273+
// fetch the root
5274+
lfs_mdir_t root;
5275+
err = lfs_dir_fetch(lfs, &root, lfs->root);
5276+
if (err) {
5277+
return err;
5278+
}
52655279

5266-
lfs_superblock_tole32(&superblock);
5267-
err = lfs_dir_commit(lfs, &root, LFS_MKATTRS(
5268-
{tag, &superblock}));
5269-
if (err) {
5270-
return err;
5271-
}
5280+
// update the superblock
5281+
lfs_superblock_t superblock;
5282+
lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0),
5283+
LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
5284+
&superblock);
5285+
if (tag < 0) {
5286+
return tag;
52725287
}
5288+
lfs_superblock_fromle32(&superblock);
5289+
5290+
superblock.block_count = lfs->block_count;
52735291

5292+
lfs_superblock_tole32(&superblock);
5293+
err = lfs_dir_commit(lfs, &root, LFS_MKATTRS(
5294+
{tag, &superblock}));
5295+
if (err) {
5296+
return err;
5297+
}
52745298
return 0;
52755299
}
52765300
#endif

‎lfs.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -766,7 +766,11 @@ int lfs_fs_gc(lfs_t *lfs);
766766
// Grows the filesystem to a new size, updating the superblock with the new
767767
// block count.
768768
//
769-
// Note: This is irreversible.
769+
// If LFS_SHRINKNONRELOCATING is defined, this function will also accept
770+
// block_counts smaller than the current configuration, after checking
771+
// that none of the blocks that are being removed are in use.
772+
// Note that littlefs's pseudorandom block allocation means that
773+
// this is very unlikely to work in the general case.
770774
//
771775
// Returns a negative error code on failure.
772776
int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count);

‎tests/test_shrink.toml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# simple shrink
2+
[cases.test_shrink_simple]
3+
defines.BLOCK_COUNT = [10, 15, 20]
4+
defines.AFTER_BLOCK_COUNT = [5, 10, 15, 19]
5+
6+
if = "AFTER_BLOCK_COUNT <= BLOCK_COUNT"
7+
code = '''
8+
#ifdef LFS_SHRINKNONRELOCATING
9+
lfs_t lfs;
10+
lfs_format(&lfs, cfg) => 0;
11+
lfs_mount(&lfs, cfg) => 0;
12+
lfs_fs_grow(&lfs, AFTER_BLOCK_COUNT) => 0;
13+
lfs_unmount(&lfs);
14+
if (BLOCK_COUNT != AFTER_BLOCK_COUNT) {
15+
lfs_mount(&lfs, cfg) => LFS_ERR_INVAL;
16+
}
17+
lfs_t lfs2 = lfs;
18+
struct lfs_config cfg2 = *cfg;
19+
cfg2.block_count = AFTER_BLOCK_COUNT;
20+
lfs2.cfg = &cfg2;
21+
lfs_mount(&lfs2, &cfg2) => 0;
22+
lfs_unmount(&lfs2) => 0;
23+
#endif
24+
'''
25+
26+
# shrinking full
27+
[cases.test_shrink_full]
28+
defines.BLOCK_COUNT = [10, 15, 20]
29+
defines.AFTER_BLOCK_COUNT = [5, 7, 10, 12, 15, 17, 20]
30+
defines.FILES_COUNT = [7, 8, 9, 10]
31+
if = "AFTER_BLOCK_COUNT <= BLOCK_COUNT && FILES_COUNT + 2 < BLOCK_COUNT"
32+
code = '''
33+
#ifdef LFS_SHRINKNONRELOCATING
34+
lfs_t lfs;
35+
lfs_format(&lfs, cfg) => 0;
36+
// create FILES_COUNT files of BLOCK_SIZE - 50 bytes (to avoid inlining)
37+
lfs_mount(&lfs, cfg) => 0;
38+
for (int i = 0; i < FILES_COUNT + 1; i++) {
39+
lfs_file_t file;
40+
char path[1024];
41+
sprintf(path, "file_%03d", i);
42+
lfs_file_open(&lfs, &file, path,
43+
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
44+
char wbuffer[BLOCK_SIZE];
45+
memset(wbuffer, 'b', BLOCK_SIZE);
46+
// Ensure one block is taken per file, but that files are not inlined.
47+
lfs_size_t size = BLOCK_SIZE - 0x40;
48+
sprintf(wbuffer, "Hi %03d", i);
49+
lfs_file_write(&lfs, &file, wbuffer, size) => size;
50+
lfs_file_close(&lfs, &file) => 0;
51+
}
52+
53+
int err = lfs_fs_grow(&lfs, AFTER_BLOCK_COUNT);
54+
if (err == 0) {
55+
for (int i = 0; i < FILES_COUNT + 1; i++) {
56+
lfs_file_t file;
57+
char path[1024];
58+
sprintf(path, "file_%03d", i);
59+
lfs_file_open(&lfs, &file, path,
60+
LFS_O_RDONLY ) => 0;
61+
lfs_size_t size = BLOCK_SIZE - 0x40;
62+
char wbuffer[size];
63+
char wbuffer_ref[size];
64+
// Ensure one block is taken per file, but that files are not inlined.
65+
memset(wbuffer_ref, 'b', size);
66+
sprintf(wbuffer_ref, "Hi %03d", i);
67+
lfs_file_read(&lfs, &file, wbuffer, BLOCK_SIZE) => size;
68+
lfs_file_close(&lfs, &file) => 0;
69+
for (lfs_size_t j = 0; j < size; j++) {
70+
wbuffer[j] => wbuffer_ref[j];
71+
}
72+
}
73+
} else {
74+
assert(err == LFS_ERR_NOTEMPTY);
75+
}
76+
77+
lfs_unmount(&lfs) => 0;
78+
if (err == 0 ) {
79+
if ( AFTER_BLOCK_COUNT != BLOCK_COUNT ) {
80+
lfs_mount(&lfs, cfg) => LFS_ERR_INVAL;
81+
}
82+
83+
lfs_t lfs2 = lfs;
84+
struct lfs_config cfg2 = *cfg;
85+
cfg2.block_count = AFTER_BLOCK_COUNT;
86+
lfs2.cfg = &cfg2;
87+
lfs_mount(&lfs2, &cfg2) => 0;
88+
for (int i = 0; i < FILES_COUNT + 1; i++) {
89+
lfs_file_t file;
90+
char path[1024];
91+
sprintf(path, "file_%03d", i);
92+
lfs_file_open(&lfs2, &file, path,
93+
LFS_O_RDONLY ) => 0;
94+
lfs_size_t size = BLOCK_SIZE - 0x40;
95+
char wbuffer[size];
96+
char wbuffer_ref[size];
97+
// Ensure one block is taken per file, but that files are not inlined.
98+
memset(wbuffer_ref, 'b', size);
99+
sprintf(wbuffer_ref, "Hi %03d", i);
100+
lfs_file_read(&lfs2, &file, wbuffer, BLOCK_SIZE) => size;
101+
lfs_file_close(&lfs2, &file) => 0;
102+
for (lfs_size_t j = 0; j < size; j++) {
103+
wbuffer[j] => wbuffer_ref[j];
104+
}
105+
}
106+
lfs_unmount(&lfs2);
107+
}
108+
#endif
109+
'''

‎tests/test_superblocks.toml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,114 @@ code = '''
524524
lfs_unmount(&lfs) => 0;
525525
'''
526526

527+
528+
# mount and grow the filesystem
529+
[cases.test_superblocks_shrink]
530+
defines.BLOCK_COUNT = 'ERASE_COUNT'
531+
defines.BLOCK_COUNT_2 = ['ERASE_COUNT/2', 'ERASE_COUNT/4', '2']
532+
defines.KNOWN_BLOCK_COUNT = [true, false]
533+
code = '''
534+
#ifdef LFS_SHRINKNONRELOCATING
535+
lfs_t lfs;
536+
lfs_format(&lfs, cfg) => 0;
537+
538+
if (KNOWN_BLOCK_COUNT) {
539+
cfg->block_count = BLOCK_COUNT;
540+
} else {
541+
cfg->block_count = 0;
542+
}
543+
544+
// mount with block_size < erase_size
545+
lfs_mount(&lfs, cfg) => 0;
546+
struct lfs_fsinfo fsinfo;
547+
lfs_fs_stat(&lfs, &fsinfo) => 0;
548+
assert(fsinfo.block_size == BLOCK_SIZE);
549+
assert(fsinfo.block_count == BLOCK_COUNT);
550+
lfs_unmount(&lfs) => 0;
551+
552+
// same size is a noop
553+
lfs_mount(&lfs, cfg) => 0;
554+
lfs_fs_grow(&lfs, BLOCK_COUNT) => 0;
555+
lfs_fs_stat(&lfs, &fsinfo) => 0;
556+
assert(fsinfo.block_size == BLOCK_SIZE);
557+
assert(fsinfo.block_count == BLOCK_COUNT);
558+
lfs_unmount(&lfs) => 0;
559+
560+
lfs_mount(&lfs, cfg) => 0;
561+
lfs_fs_stat(&lfs, &fsinfo) => 0;
562+
assert(fsinfo.block_size == BLOCK_SIZE);
563+
assert(fsinfo.block_count == BLOCK_COUNT);
564+
lfs_unmount(&lfs) => 0;
565+
566+
// grow to new size
567+
lfs_mount(&lfs, cfg) => 0;
568+
lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0;
569+
lfs_fs_stat(&lfs, &fsinfo) => 0;
570+
assert(fsinfo.block_size == BLOCK_SIZE);
571+
assert(fsinfo.block_count == BLOCK_COUNT_2);
572+
lfs_unmount(&lfs) => 0;
573+
574+
if (KNOWN_BLOCK_COUNT) {
575+
cfg->block_count = BLOCK_COUNT_2;
576+
} else {
577+
cfg->block_count = 0;
578+
}
579+
580+
lfs_mount(&lfs, cfg) => 0;
581+
lfs_fs_stat(&lfs, &fsinfo) => 0;
582+
assert(fsinfo.block_size == BLOCK_SIZE);
583+
assert(fsinfo.block_count == BLOCK_COUNT_2);
584+
lfs_unmount(&lfs) => 0;
585+
586+
// mounting with the previous size should fail
587+
cfg->block_count = BLOCK_COUNT;
588+
lfs_mount(&lfs, cfg) => LFS_ERR_INVAL;
589+
590+
if (KNOWN_BLOCK_COUNT) {
591+
cfg->block_count = BLOCK_COUNT_2;
592+
} else {
593+
cfg->block_count = 0;
594+
}
595+
596+
// same size is a noop
597+
lfs_mount(&lfs, cfg) => 0;
598+
lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0;
599+
lfs_fs_stat(&lfs, &fsinfo) => 0;
600+
assert(fsinfo.block_size == BLOCK_SIZE);
601+
assert(fsinfo.block_count == BLOCK_COUNT_2);
602+
lfs_unmount(&lfs) => 0;
603+
604+
lfs_mount(&lfs, cfg) => 0;
605+
lfs_fs_stat(&lfs, &fsinfo) => 0;
606+
assert(fsinfo.block_size == BLOCK_SIZE);
607+
assert(fsinfo.block_count == BLOCK_COUNT_2);
608+
lfs_unmount(&lfs) => 0;
609+
610+
// do some work
611+
lfs_mount(&lfs, cfg) => 0;
612+
lfs_fs_stat(&lfs, &fsinfo) => 0;
613+
assert(fsinfo.block_size == BLOCK_SIZE);
614+
assert(fsinfo.block_count == BLOCK_COUNT_2);
615+
lfs_file_t file;
616+
lfs_file_open(&lfs, &file, "test",
617+
LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0;
618+
lfs_file_write(&lfs, &file, "hello!", 6) => 6;
619+
lfs_file_close(&lfs, &file) => 0;
620+
lfs_unmount(&lfs) => 0;
621+
622+
lfs_mount(&lfs, cfg) => 0;
623+
lfs_fs_stat(&lfs, &fsinfo) => 0;
624+
assert(fsinfo.block_size == BLOCK_SIZE);
625+
assert(fsinfo.block_count == BLOCK_COUNT_2);
626+
lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0;
627+
uint8_t buffer[256];
628+
lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6;
629+
lfs_file_close(&lfs, &file) => 0;
630+
assert(memcmp(buffer, "hello!", 6) == 0);
631+
lfs_unmount(&lfs) => 0;
632+
#endif
633+
'''
634+
527635
# test that metadata_max does not cause problems for superblock compaction
528636
[cases.test_superblocks_metadata_max]
529637
defines.METADATA_MAX = [

0 commit comments

Comments
 (0)