From 49698e431f3197b07a5de22f0592a674072d03fe Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 3 Mar 2018 10:26:06 -0600 Subject: [PATCH 01/31] Separated type/struct fields in dir entries The separation of data-structure vs entry type has been implicit for a while now, and even taken advantage of to simplify the traverse logic. Explicitely separating the data-struct and entry types allows us to introduce new data structures (inlined files). --- lfs.c | 58 +++++++++++++++++++++++++++++----------------------------- lfs.h | 12 +++++++++--- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/lfs.c b/lfs.c index 3aef9fcc..533b1ea7 100644 --- a/lfs.c +++ b/lfs.c @@ -793,7 +793,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // special case for root dir if (pathname[0] == '\0') { *entry = (lfs_entry_t){ - .d.type = LFS_TYPE_DIR, + .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, .d.elen = sizeof(entry->d) - 4, .d.alen = 0, .d.nlen = 0, @@ -844,8 +844,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (((0x7f & entry->d.type) != LFS_TYPE_REG && - (0x7f & entry->d.type) != LFS_TYPE_DIR) || + if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && + (0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || entry->d.nlen != pathlen) { continue; } @@ -864,13 +864,13 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } // check that entry has not been moved - if (entry->d.type & 0x80) { + if (entry->d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry->d.u); if (moved < 0 || moved) { return (moved < 0) ? moved : LFS_ERR_NOENT; } - entry->d.type &= ~0x80; + entry->d.type &= ~LFS_STRUCT_MOVED; } pathname += pathlen; @@ -880,7 +880,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } // continue on if we hit a directory - if (entry->d.type != LFS_TYPE_DIR) { + if ((0xf & entry->d.type) != LFS_TYPE_DIR) { return LFS_ERR_NOTDIR; } @@ -931,7 +931,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err; } - entry.d.type = LFS_TYPE_DIR; + entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; entry.d.nlen = strlen(path); @@ -963,7 +963,7 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) { err = lfs_dir_find(lfs, dir, &entry, &path); if (err) { return err; - } else if (entry.d.type != LFS_TYPE_DIR) { + } else if (entry.d.type != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { return LFS_ERR_NOTDIR; } @@ -1021,13 +1021,13 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return (err == LFS_ERR_NOENT) ? 0 : err; } - if ((0x7f & entry.d.type) != LFS_TYPE_REG && - (0x7f & entry.d.type) != LFS_TYPE_DIR) { + if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && + (0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { continue; } // check that entry has not been moved - if (entry.d.type & 0x80) { + if (entry.d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry.d.u); if (moved < 0) { return moved; @@ -1037,13 +1037,13 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { continue; } - entry.d.type &= ~0x80; + entry.d.type &= ~LFS_STRUCT_MOVED; } break; } - info->type = entry.d.type; + info->type = 0xf & entry.d.type; if (info->type == LFS_TYPE_REG) { info->size = entry.d.u.file.size; } @@ -1319,7 +1319,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // create entry to remember name - entry.d.type = LFS_TYPE_REG; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; entry.d.nlen = strlen(path); @@ -1329,7 +1329,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, if (err) { return err; } - } else if (entry.d.type == LFS_TYPE_DIR) { + } else if ((0xf & entry.d.type) == LFS_TYPE_DIR) { return LFS_ERR_ISDIR; } else if (flags & LFS_O_EXCL) { return LFS_ERR_EXIST; @@ -1537,7 +1537,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - LFS_ASSERT(entry.d.type == LFS_TYPE_REG); + LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; @@ -1826,7 +1826,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { } memset(info, 0, sizeof(*info)); - info->type = entry.d.type; + info->type = 0xf & entry.d.type; if (info->type == LFS_TYPE_REG) { info->size = entry.d.u.file.size; } @@ -1867,7 +1867,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } lfs_dir_t dir; - if (entry.d.type == LFS_TYPE_DIR) { + if ((0xf & entry.d.type) == LFS_TYPE_DIR) { // must be empty before removal, checking size // without masking top bit checks for any case where // dir is not empty @@ -1886,7 +1886,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // if we were a directory, find pred, replace tail - if (entry.d.type == LFS_TYPE_DIR) { + if ((0xf & entry.d.type) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &cwd); if (res < 0) { return res; @@ -1949,7 +1949,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } lfs_dir_t dir; - if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { // must be empty before removal, checking size // without masking top bit checks for any case where // dir is not empty @@ -1962,7 +1962,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // mark as moving - oldentry.d.type |= 0x80; + oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); if (err) { return err; @@ -1976,7 +1976,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // move to new location lfs_entry_t newentry = preventry; newentry.d = oldentry.d; - newentry.d.type &= ~0x80; + newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); if (prevexists) { @@ -2003,7 +2003,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // if we were a directory, find pred, replace tail - if (prevexists && preventry.d.type == LFS_TYPE_DIR) { + if (prevexists && (0xf & preventry.d.type) == LFS_TYPE_DIR) { int res = lfs_pred(lfs, dir.pair, &newcwd); if (res < 0) { return res; @@ -2134,7 +2134,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write superblocks lfs_superblock_t superblock = { .off = sizeof(superdir.d), - .d.type = LFS_TYPE_SUPERBLOCK, + .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, .d.nlen = sizeof(superblock.d.magic), .d.version = LFS_DISK_VERSION, @@ -2263,7 +2263,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } dir.off += lfs_entry_size(&entry); - if ((0x70 & entry.d.type) == (0x70 & LFS_TYPE_REG)) { + if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); if (err) { @@ -2353,7 +2353,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], break; } - if (((0x70 & entry->d.type) == (0x70 & LFS_TYPE_DIR)) && + if (((0x70 & entry->d.type) == LFS_STRUCT_DIR) && lfs_paircmp(entry->d.u.dir, dir) == 0) { return true; } @@ -2393,7 +2393,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { break; } - if (!(0x80 & entry.d.type) && + if (!(LFS_STRUCT_MOVED & entry.d.type) && memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { return true; } @@ -2525,7 +2525,7 @@ int lfs_deorphan(lfs_t *lfs) { } // found moved entry - if (entry.d.type & 0x80) { + if (entry.d.type & LFS_STRUCT_MOVED) { int moved = lfs_moved(lfs, &entry.d.u); if (moved < 0) { return moved; @@ -2541,7 +2541,7 @@ int lfs_deorphan(lfs_t *lfs) { } else { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); - entry.d.type &= ~0x80; + entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &cwd, &entry, NULL); if (err) { return err; diff --git a/lfs.h b/lfs.h index 376776f6..4a12c4a2 100644 --- a/lfs.h +++ b/lfs.h @@ -74,9 +74,15 @@ enum lfs_error { // File types enum lfs_type { - LFS_TYPE_REG = 0x11, - LFS_TYPE_DIR = 0x22, - LFS_TYPE_SUPERBLOCK = 0x2e, + // file type + LFS_TYPE_REG = 0x01, + LFS_TYPE_DIR = 0x02, + LFS_TYPE_SUPERBLOCK = 0x0e, + + // on disk structure + LFS_STRUCT_CTZ = 0x10, + LFS_STRUCT_DIR = 0x20, + LFS_STRUCT_MOVED = 0x80, }; // File open flags From 4c35c8655af3400237940f26879cd16b6094f0a2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 3 Mar 2018 22:14:19 -0600 Subject: [PATCH 02/31] Added different sources for commits, now with disk->disk moves Previously, commits could only come from memory in RAM. This meant any entries had to be buffered in their entirety before they could be moved to a different directory pair. By adding parameters for specifying commits from existing entries stored on disk, we allow any sized entries to be moved between directory pairs with a fixed RAM cost. --- lfs.c | 85 ++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/lfs.c b/lfs.c index 533b1ea7..3bedd0df 100644 --- a/lfs.c +++ b/lfs.c @@ -486,7 +486,18 @@ static int lfs_dir_fetch(lfs_t *lfs, struct lfs_region { lfs_off_t oldoff; lfs_size_t oldlen; - const void *newdata; + + enum lfs_region_source { + LFS_FROM_MEM, + LFS_FROM_DISK, + } source; + union { + const void *mem; + struct { + lfs_block_t block; + lfs_off_t off; + } disk; + } u; lfs_size_t newlen; }; @@ -527,42 +538,49 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } int i = 0; + int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { + while (i < count && oldoff == regions[i].oldoff && + j == regions[i].newlen) { + oldoff += regions[i].oldlen; + i += 1; + j = 0; + } + + uint8_t data; if (i < count && regions[i].oldoff == oldoff) { - lfs_crc(&crc, regions[i].newdata, regions[i].newlen); - err = lfs_bd_prog(lfs, dir->pair[0], - newoff, regions[i].newdata, regions[i].newlen); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + if (regions[i].source == LFS_FROM_DISK) { + err = lfs_bd_read(lfs, regions[i].u.disk.block, + regions[i].u.disk.off + j, &data, 1); + if (err) { + return err; } - return err; + } else { + data = ((const uint8_t *)regions[i].u.mem)[j]; } - oldoff += regions[i].oldlen; - newoff += regions[i].newlen; - i += 1; + j += 1; } else { - uint8_t data; err = lfs_bd_read(lfs, oldpair[1], oldoff, &data, 1); if (err) { return err; } - lfs_crc(&crc, &data, 1); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - oldoff += 1; - newoff += 1; } + + lfs_crc(&crc, &data, 1); + err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + newoff += 1; } crc = lfs_tole32(crc); @@ -643,8 +661,10 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)}, - {entry->off+sizeof(entry->d), entry->d.nlen, data, entry->d.nlen} + {entry->off, sizeof(entry->d), + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off+sizeof(entry->d), entry->d.nlen, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, data ? 2 : 1); lfs_entry_fromle32(&entry->d); return err; @@ -659,8 +679,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, &entry->d, sizeof(entry->d)}, - {entry->off, 0, data, entry->d.nlen} + {entry->off, 0, + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off, 0, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, 2); lfs_entry_fromle32(&entry->d); return err; @@ -679,8 +701,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, &entry->d, sizeof(entry->d)}, - {entry->off, 0, data, entry->d.nlen} + {entry->off, 0, + LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, + {entry->off, 0, + LFS_FROM_MEM, {.mem = data}, entry->d.nlen} }, 2); lfs_entry_fromle32(&entry->d); if (err) { @@ -720,7 +744,8 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { // shift out the entry int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, lfs_entry_size(entry), NULL, 0}, + {entry->off, lfs_entry_size(entry), + LFS_FROM_MEM, {.mem = NULL}, 0}, }, 1); if (err) { return err; @@ -2153,7 +2178,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ {sizeof(superdir.d), sizeof(superblock.d), - &superblock.d, sizeof(superblock.d)} + LFS_FROM_MEM, {.mem = &superblock.d}, sizeof(superblock.d)} }, 1); if (err && err != LFS_ERR_CORRUPT) { return err; From 73d29f05b24765dd171f07dd41df09f4f067dc9f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 18:27:25 -0600 Subject: [PATCH 03/31] Adopted a tiny LISP-like DSL for some extra flexibility Really all this means is that the internal commit function was changed from taking an array of "commit structures" to a linked-list of "commit structures". The benefit of a linked-list is that layers of commit functions can pull off some minor modifications to the description of the commit. Most notably, commit functions can add additional entries that will be atomically written out and CRCed along with the initial commit. Also a minor benefit, this is one less parameter when committing a directory with zero entries. --- lfs.c | 114 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 54 deletions(-) diff --git a/lfs.c b/lfs.c index 3bedd0df..bc9ae58e 100644 --- a/lfs.c +++ b/lfs.c @@ -484,32 +484,36 @@ static int lfs_dir_fetch(lfs_t *lfs, } struct lfs_region { - lfs_off_t oldoff; - lfs_size_t oldlen; - - enum lfs_region_source { + enum { LFS_FROM_MEM, LFS_FROM_DISK, } source; + + lfs_off_t oldoff; + lfs_size_t oldlen; union { - const void *mem; + struct { + const void *data; + } m; struct { lfs_block_t block; lfs_off_t off; - } disk; + } d; } u; lfs_size_t newlen; + + struct lfs_region *next; }; static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - const struct lfs_region *regions, int count) { + struct lfs_region *region) { // increment revision count dir->d.rev += 1; // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (int i = 0; i < count; i++) { - dir->d.size += regions[i].newlen - regions[i].oldlen; + for (struct lfs_region *r = region; r; r = r->next) { + dir->d.size += r->newlen - r->oldlen; } const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; @@ -537,28 +541,27 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - int i = 0; + struct lfs_region *r = region; int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { - while (i < count && oldoff == regions[i].oldoff && - j == regions[i].newlen) { - oldoff += regions[i].oldlen; - i += 1; + while (r && r->oldoff == oldoff && r->newlen == j) { + oldoff += r->oldlen; + r = r->next; j = 0; } uint8_t data; - if (i < count && regions[i].oldoff == oldoff) { - if (regions[i].source == LFS_FROM_DISK) { - err = lfs_bd_read(lfs, regions[i].u.disk.block, - regions[i].u.disk.off + j, &data, 1); + if (r && r->oldoff == oldoff) { + if (r->source == LFS_FROM_DISK) { + err = lfs_bd_read(lfs, r->u.d.block, + r->u.d.off + j, &data, 1); if (err) { return err; } } else { - data = ((const uint8_t *)regions[i].u.mem)[j]; + data = ((const uint8_t *)r->u.m.data)[j]; } j += 1; @@ -660,12 +663,15 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, sizeof(entry->d), - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off+sizeof(entry->d), entry->d.nlen, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, data ? 2 : 1); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, sizeof(entry->d), + {.m.data = &entry->d}, sizeof(entry->d), + data ? + &(struct lfs_region){ + LFS_FROM_MEM, entry->off+sizeof(entry->d), entry->d.nlen, + {.m.data = data}, entry->d.nlen} + : NULL}); lfs_entry_fromle32(&entry->d); return err; } @@ -678,12 +684,13 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off, 0, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, 2); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = data}, entry->d.nlen}}); lfs_entry_fromle32(&entry->d); return err; } @@ -700,12 +707,13 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[1] = olddir.d.tail[1]; entry->off = dir->d.size - 4; lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, 0, - LFS_FROM_MEM, {.mem = &entry->d}, sizeof(entry->d)}, - {entry->off, 0, - LFS_FROM_MEM, {.mem = data}, entry->d.nlen} - }, 2); + err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = data}, entry->d.nlen}}); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -714,7 +722,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, olddir.d.size |= 0x80000000; olddir.d.tail[0] = dir->pair[0]; olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL, 0); + return lfs_dir_commit(lfs, &olddir, NULL); } int err = lfs_dir_fetch(lfs, dir, dir->d.tail); @@ -738,15 +746,14 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL, 0); + return lfs_dir_commit(lfs, &pdir, NULL); } } // shift out the entry - int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {entry->off, lfs_entry_size(entry), - LFS_FROM_MEM, {.mem = NULL}, 0}, - }, 1); + int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ + LFS_FROM_MEM, entry->off, lfs_entry_size(entry), + {.m.data = NULL}, 0}); if (err) { return err; } @@ -951,7 +958,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &dir, NULL, 0); + err = lfs_dir_commit(lfs, &dir, NULL); if (err) { return err; } @@ -1921,7 +1928,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &cwd, NULL, 0); + err = lfs_dir_commit(lfs, &cwd, NULL); if (err) { return err; } @@ -2038,7 +2045,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &newcwd, NULL, 0); + err = lfs_dir_commit(lfs, &newcwd, NULL); if (err) { return err; } @@ -2148,7 +2155,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit(lfs, &root, NULL, 0); + err = lfs_dir_commit(lfs, &root, NULL); if (err) { return err; } @@ -2176,10 +2183,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_tole32(&superblock.d); bool valid = false; for (int i = 0; i < 2; i++) { - err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ - {sizeof(superdir.d), sizeof(superblock.d), - LFS_FROM_MEM, {.mem = &superblock.d}, sizeof(superblock.d)} - }, 1); + err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ + LFS_FROM_MEM, sizeof(superdir.d), sizeof(superblock.d), + {.m.data = &superblock.d}, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2470,7 +2476,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, NULL, 0); + return lfs_dir_commit(lfs, &parent, NULL); } // couldn't find dir, must be new @@ -2512,7 +2518,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL); if (err) { return err; } @@ -2528,7 +2534,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dir_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL); if (err) { return err; } From e3daee26218562d55e939e6ce2ffa97149d12eb4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 18:27:43 -0600 Subject: [PATCH 04/31] Changed dir append to mirror commit DSL Expiremental implementation. This opens up the opportunity to use the same commit description for both commits and appends, which effectively do the same thing. This should lead to better code reuse. --- lfs.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index bc9ae58e..3c7fb493 100644 --- a/lfs.c +++ b/lfs.c @@ -676,6 +676,59 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, return err; } +static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *region) { + // check if we fit, if top bit is set we do not and move on + while (true) { + if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { + entry->off = dir->d.size - 4; + for (struct lfs_region *r = region; r; r = r->next) { + r->oldoff += entry->off; + } + + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, 4, region}); + lfs_entry_fromle32(&entry->d); + return err; + } + + // we need to allocate a new dir block + if (!(0x80000000 & dir->d.size)) { + lfs_dir_t olddir = *dir; + int err = lfs_dir_alloc(lfs, dir); + if (err) { + return err; + } + + dir->d.tail[0] = olddir.d.tail[0]; + dir->d.tail[1] = olddir.d.tail[1]; + entry->off = dir->d.size - 4; + lfs_entry_tole32(&entry->d); + err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, 4, region}); + lfs_entry_fromle32(&entry->d); + if (err) { + return err; + } + + olddir.d.size |= 0x80000000; + olddir.d.tail[0] = dir->pair[0]; + olddir.d.tail[1] = dir->pair[1]; + return lfs_dir_commit(lfs, &olddir, NULL); + } + + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + if (err) { + return err; + } + } +} + static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const void *data) { // check if we fit, if top bit is set we do not and move on @@ -973,7 +1026,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - err = lfs_dir_append(lfs, &cwd, &entry, path); + err = lfs_dir_append_(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = path}, entry.d.nlen}}); if (err) { return err; } @@ -1357,7 +1416,13 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; - err = lfs_dir_append(lfs, &cwd, &entry, path); + err = lfs_dir_append_(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = path}, entry.d.nlen}}); if (err) { return err; } @@ -2017,7 +2082,13 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } } else { - err = lfs_dir_append(lfs, &newcwd, &newentry, newpath); + err = lfs_dir_append_(lfs, &newcwd, &newentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (uint8_t*)&newentry.d + 4}, sizeof(newentry.d) - 4, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; } From 692f0c542eb21816fb0a548b392e863f951138bb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 19:27:57 -0600 Subject: [PATCH 05/31] Naive implementation of resizable entries Now, with the off, diff, and len parameters in each commit entry, we can build up directory commits that resize entries. This adds complexity but opens up the directory blocks to be much more flexible. The main concern is that resizing entries can push around neighboring entries in surprising ways, such as pushing them into new directory blocks when a directory splits. This can break littlefs's internal logic in how it tracks in-flight entries. The most problematic example being open files. Fortunately, this is helped by a global linked-list of all files and directories opened by the filesystem. As entries change size, the state of open files/dirs may be updated as needed. Note this already needed to exist for the ability to remove files/dirs, which has the same issue. --- lfs.c | 225 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 131 insertions(+), 94 deletions(-) diff --git a/lfs.c b/lfs.c index 3c7fb493..cf8f2e11 100644 --- a/lfs.c +++ b/lfs.c @@ -489,8 +489,8 @@ struct lfs_region { LFS_FROM_DISK, } source; - lfs_off_t oldoff; - lfs_size_t oldlen; + lfs_off_t off; + lfs_ssize_t diff; union { struct { const void *data; @@ -500,20 +500,20 @@ struct lfs_region { lfs_off_t off; } d; } u; - lfs_size_t newlen; + lfs_size_t len; struct lfs_region *next; }; static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - struct lfs_region *region) { + struct lfs_region *regions) { // increment revision count dir->d.rev += 1; // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (struct lfs_region *r = region; r; r = r->next) { - dir->d.size += r->newlen - r->oldlen; + for (struct lfs_region *r = regions; r; r = r->next) { + dir->d.size += r->diff; } const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; @@ -541,19 +541,19 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_region *r = region; + struct lfs_region *r = regions; int j = 0; lfs_off_t oldoff = sizeof(dir->d); lfs_off_t newoff = sizeof(dir->d); while (newoff < (0x7fffffff & dir->d.size)-4) { - while (r && r->oldoff == oldoff && r->newlen == j) { - oldoff += r->oldlen; + while (r && r->off == oldoff && r->len == j) { + oldoff += r->len - r->diff; r = r->next; j = 0; } uint8_t data; - if (r && r->oldoff == oldoff) { + if (r && r->off == oldoff) { if (r->source == LFS_FROM_DISK) { err = lfs_bd_read(lfs, r->u.d.block, r->u.d.off + j, &data, 1); @@ -660,37 +660,21 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, const void *data) { - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, sizeof(entry->d), - {.m.data = &entry->d}, sizeof(entry->d), - data ? - &(struct lfs_region){ - LFS_FROM_MEM, entry->off+sizeof(entry->d), entry->d.nlen, - {.m.data = data}, entry->d.nlen} - : NULL}); - lfs_entry_fromle32(&entry->d); - return err; -} - -static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *region) { +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { entry->off = dir->d.size - 4; - for (struct lfs_region *r = region; r; r = r->next) { - r->oldoff += entry->off; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; } lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, 4, region}); + LFS_FROM_MEM, entry->off, +4, + {.m.data = &entry->d}, 4, regions}); lfs_entry_fromle32(&entry->d); return err; } @@ -706,11 +690,15 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; entry->off = dir->d.size - 4; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; + } + lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, 4, region}); + LFS_FROM_MEM, entry->off, +4, + {.m.data = &entry->d}, 4, regions}); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -729,60 +717,95 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, } } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, const void *data) { - // check if we fit, if top bit is set we do not and move on - while (true) { - if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; +static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { + lfs_off_t oldoff = entry->off; + lfs_ssize_t diff = 0; + lfs_size_t len = 0; + struct lfs_region **tail = ®ions; + for (struct lfs_region *r = regions; r; r = r->next) { + diff += r->diff; + len += r->len; + tail = &r->next; + } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = data}, entry->d.nlen}}); - lfs_entry_fromle32(&entry->d); - return err; + // do we still fit? + if ((0x7fffffff & dir->d.size) + diff <= lfs->cfg->block_size) { + for (struct lfs_region *r = regions; r; r = r->next) { + r->off += entry->off; } - // we need to allocate a new dir block - if (!(0x80000000 & dir->d.size)) { - lfs_dir_t olddir = *dir; - int err = lfs_dir_alloc(lfs, dir); - if (err) { - return err; - } + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + LFS_FROM_MEM, entry->off, 0, + {.m.data = &entry->d}, sizeof(entry->d), regions}); + lfs_entry_fromle32(&entry->d); + if (err) { + return err; + } + } else { + lfs_dir_t olddir = *dir; - dir->d.tail[0] = olddir.d.tail[0]; - dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = data}, entry->d.nlen}}); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } + // mark as moving + entry->d.type |= LFS_STRUCT_MOVED; + int err = lfs_dir_commit(lfs, &olddir, + &(struct lfs_region){ + LFS_FROM_MEM, oldoff, 0, + {.m.data = &entry->d.type}, 1}); + if (err) { + return err; + } - olddir.d.size |= 0x80000000; - olddir.d.tail[0] = dir->pair[0]; - olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL); + // append updated entry + entry->d.type &= LFS_STRUCT_MOVED; + lfs_size_t prefix = regions->off; + lfs_size_t suffix = lfs_entry_size(entry) - len - 4; + for (struct lfs_region *r = regions; r; r = r->next) { + r->off = 0; + r->diff = r->len; } - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); + regions = &(struct lfs_region){ + LFS_FROM_DISK, 0, +prefix, + {.d.block = olddir.pair[0], .d.off = entry->off}, + prefix, regions}; + *tail = &(struct lfs_region){ + LFS_FROM_DISK, 0, +suffix, + {.d.block = olddir.pair[0], .d.off = entry->off+prefix+len}, + suffix}; + + // remove old entry + err = lfs_dir_commit(lfs, &olddir, + &(struct lfs_region){ + LFS_FROM_MEM, oldoff, -(lfs_entry_size(entry) - diff), + {.m.data = NULL}, 0}); if (err) { return err; } } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff == oldoff) { + f->poff = entry->off; + } else if (f->poff > entry->off) { + f->poff += diff; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > entry->off) { + d->off += diff; + d->pos += diff; + } + } + } + + return 0; } static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { @@ -805,7 +828,7 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { // shift out the entry int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, lfs_entry_size(entry), + LFS_FROM_MEM, entry->off, -lfs_entry_size(entry), {.m.data = NULL}, 0}); if (err) { return err; @@ -1026,12 +1049,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - err = lfs_dir_append_(lfs, &cwd, &entry, + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), + {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); if (err) { return err; @@ -1416,12 +1439,12 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; - err = lfs_dir_append_(lfs, &cwd, &entry, + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&entry.d + 4}, sizeof(entry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), + {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); if (err) { return err; @@ -1638,7 +1661,10 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - err = lfs_dir_update(lfs, &cwd, &entry, NULL); + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t *)&entry.d+4}, sizeof(entry.d)-4}); if (err) { return err; } @@ -2077,17 +2103,25 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d.nlen = strlen(newpath); if (prevexists) { - err = lfs_dir_update(lfs, &newcwd, &newentry, newpath); + err = lfs_dir_update(lfs, &newcwd, &newentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t*)&newentry.d+4}, + sizeof(newentry.d)-4, + &(struct lfs_region){ + LFS_FROM_MEM, sizeof(newentry.d)-4, 0, + {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; } } else { - err = lfs_dir_append_(lfs, &newcwd, &newentry, + err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = (uint8_t*)&newentry.d + 4}, sizeof(newentry.d) - 4, + LFS_FROM_MEM, 0, +(sizeof(newentry.d)-4), + {.m.data = (const uint8_t*)&newentry.d+4}, + sizeof(newentry.d)-4, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, + LFS_FROM_MEM, 0, +newentry.d.nlen, {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; @@ -2255,7 +2289,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { bool valid = false; for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(superdir.d), sizeof(superblock.d), + LFS_FROM_MEM, sizeof(superdir.d), 0, {.m.data = &superblock.d}, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; @@ -2520,7 +2554,10 @@ static int lfs_relocate(lfs_t *lfs, entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; - int err = lfs_dir_update(lfs, &parent, &entry, NULL); + int err = lfs_dir_update(lfs, &parent, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = (const uint8_t*)&entry.d+4}, sizeof(entry.d)-4}); if (err) { return err; } From ca3d6a52d27482a3520294f91626c78dc5051b0e Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 10 Mar 2018 21:28:06 -0600 Subject: [PATCH 06/31] Made implicity tag updates explicit Before, tags were implicitly updated by the dir update functions, which have a strong understanding of the entry struct. However, most of the time the tag was already a part of the entry struct being committed. By making tag updates explicit, this does add cost to commits that now have to pass tag updates explicitly, but it reduces cost where that tag and entry update can be combined into one commit region. It also simplifies the dir update functions. --- lfs.c | 63 ++++++++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 37 deletions(-) diff --git a/lfs.c b/lfs.c index cf8f2e11..0f4ab2c1 100644 --- a/lfs.c +++ b/lfs.c @@ -349,10 +349,11 @@ static void lfs_entry_fromle32(struct lfs_disk_entry *d) { d->u.dir[1] = lfs_fromle32(d->u.dir[1]); } -static void lfs_entry_tole32(struct lfs_disk_entry *d) { - d->u.dir[0] = lfs_tole32(d->u.dir[0]); - d->u.dir[1] = lfs_tole32(d->u.dir[1]); -} +// TODO +//static void lfs_entry_tole32(struct lfs_disk_entry *d) { +// d->u.dir[0] = lfs_tole32(d->u.dir[0]); +// d->u.dir[1] = lfs_tole32(d->u.dir[1]); +//} static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->root[0] = lfs_fromle32(d->root[0]); @@ -670,13 +671,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, +4, - {.m.data = &entry->d}, 4, regions}); - lfs_entry_fromle32(&entry->d); - return err; + return lfs_dir_commit(lfs, dir, regions); } // we need to allocate a new dir block @@ -694,12 +689,7 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, +4, - {.m.data = &entry->d}, 4, regions}); - lfs_entry_fromle32(&entry->d); + err = lfs_dir_commit(lfs, dir, regions); if (err) { return err; } @@ -735,12 +725,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - LFS_FROM_MEM, entry->off, 0, - {.m.data = &entry->d}, sizeof(entry->d), regions}); - lfs_entry_fromle32(&entry->d); + int err = lfs_dir_commit(lfs, dir, regions); if (err) { return err; } @@ -1051,8 +1036,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), - {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, + LFS_FROM_MEM, 0, +sizeof(entry.d), + {.m.data = &entry.d}, sizeof(entry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); @@ -1441,8 +1426,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.u.file.size = 0; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(entry.d)-4), - {.m.data = (uint8_t*)&entry.d+4}, sizeof(entry.d)-4, + LFS_FROM_MEM, 0, +sizeof(entry.d), + {.m.data = &entry.d}, sizeof(entry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +entry.d.nlen, {.m.data = path}, entry.d.nlen}}); @@ -1664,7 +1649,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t *)&entry.d+4}, sizeof(entry.d)-4}); + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } @@ -2086,7 +2071,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); + err = lfs_dir_update(lfs, &oldcwd, &oldentry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = &oldentry.d}, sizeof(oldentry.d)}); if (err) { return err; } @@ -2106,10 +2094,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t*)&newentry.d+4}, - sizeof(newentry.d)-4, + {.m.data = &newentry.d}, sizeof(newentry.d), &(struct lfs_region){ - LFS_FROM_MEM, sizeof(newentry.d)-4, 0, + LFS_FROM_MEM, sizeof(newentry.d), 0, {.m.data = newpath}, newentry.d.nlen}}); if (err) { return err; @@ -2117,9 +2104,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +(sizeof(newentry.d)-4), - {.m.data = (const uint8_t*)&newentry.d+4}, - sizeof(newentry.d)-4, + LFS_FROM_MEM, 0, +sizeof(newentry.d), + {.m.data = &newentry.d}, sizeof(newentry.d), &(struct lfs_region){ LFS_FROM_MEM, 0, +newentry.d.nlen, {.m.data = newpath}, newentry.d.nlen}}); @@ -2557,7 +2543,7 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ LFS_FROM_MEM, 0, 0, - {.m.data = (const uint8_t*)&entry.d+4}, sizeof(entry.d)-4}); + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } @@ -2681,7 +2667,10 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &cwd, &entry, NULL); + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + LFS_FROM_MEM, 0, 0, + {.m.data = &entry.d}, sizeof(entry.d)}); if (err) { return err; } From f30ab677a4553f6da441963444d1fec08c9df2b8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 11 Mar 2018 11:28:13 -0500 Subject: [PATCH 07/31] Traded enum-based DSL for full callback-based DSL Now, instead of passing an enum for mem/disk commits, we pass a function pointer that can specify any behaviour. This has the benefit of opening up the possibility to pass any sort of commit logic to the committers, and unused logic can be garbage-collected by the compiler if unused. The downside is that unfortunately compilers have a harder time optimizing around functions pointers than enums, and fitting the state into structs for the callbacks may be costly. --- lfs.c | 274 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 151 insertions(+), 123 deletions(-) diff --git a/lfs.c b/lfs.c index 0f4ab2c1..35bb1646 100644 --- a/lfs.c +++ b/lfs.c @@ -484,30 +484,85 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -struct lfs_region { - enum { - LFS_FROM_MEM, - LFS_FROM_DISK, - } source; +struct lfs_commit { + uint32_t crc; + lfs_block_t block; + lfs_off_t off; +}; +static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_size_t size) { + lfs_crc(&c->crc, data, size); + int err = lfs_bd_prog(lfs, c->block, c->off, data, size); + c->off += size; + return err; +} + +struct lfs_region { lfs_off_t off; lfs_ssize_t diff; - union { - struct { - const void *data; - } m; - struct { - lfs_block_t block; - lfs_off_t off; - } d; - } u; - lfs_size_t len; - + int (*commit)(lfs_t *lfs, struct lfs_commit *c, const void *p); + const void *data; struct lfs_region *next; }; +struct lfs_commit_mem { + const void *data; + lfs_size_t size; +}; + +static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, const void *p) { + const struct lfs_commit_mem *m = p; + return lfs_commit(lfs, c, m->data, m->size); +} + +struct lfs_commit_disk { + lfs_block_t block; + lfs_off_t off; + lfs_size_t size; + struct lfs_region *regions; +}; + +static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, const void *p) { + const struct lfs_commit_disk *u = p; + + struct lfs_region *r = u->regions; + lfs_off_t off = 0; + while (true) { + if (r && r->off == off) { + lfs_off_t orig = c->off; + int err = r->commit(lfs, c, r->data); + if (err) { + return err; + } + + off += (c->off - orig) - r->diff; + r = r->next; + } else if (off < u->size) { + uint8_t data; + int err = lfs_bd_read(lfs, u->block, u->off + off, &data, 1); + if (err) { + return err; + } + + err = lfs_commit(lfs, c, &data, 1); + if (err) { + return err; + } + + off += 1; + } else { + return 0; + } + } +} + static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, struct lfs_region *regions) { + // state for copying over + const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; + lfs_size_t oldsize = (0x7fffffff & dir->d.size) - 4; + bool relocated = false; + // increment revision count dir->d.rev += 1; @@ -517,9 +572,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, dir->d.size += r->diff; } - const lfs_block_t oldpair[2] = {dir->pair[0], dir->pair[1]}; - bool relocated = false; - while (true) { if (true) { int err = lfs_bd_erase(lfs, dir->pair[0]); @@ -530,10 +582,20 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - uint32_t crc = 0xffffffff; + struct lfs_commit c = { + .crc = 0xffffffff, + .block = dir->pair[0], + .off = 0, + }; + lfs_dir_tole32(&dir->d); - lfs_crc(&crc, &dir->d, sizeof(dir->d)); - err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); + err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ + oldpair[1], 0, oldsize, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &dir->d, sizeof(dir->d)}, + regions}}); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -542,54 +604,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_region *r = regions; - int j = 0; - lfs_off_t oldoff = sizeof(dir->d); - lfs_off_t newoff = sizeof(dir->d); - while (newoff < (0x7fffffff & dir->d.size)-4) { - while (r && r->off == oldoff && r->len == j) { - oldoff += r->len - r->diff; - r = r->next; - j = 0; - } - - uint8_t data; - if (r && r->off == oldoff) { - if (r->source == LFS_FROM_DISK) { - err = lfs_bd_read(lfs, r->u.d.block, - r->u.d.off + j, &data, 1); - if (err) { - return err; - } - } else { - data = ((const uint8_t *)r->u.m.data)[j]; - } - - j += 1; - } else { - err = lfs_bd_read(lfs, oldpair[1], oldoff, &data, 1); - if (err) { - return err; - } - - oldoff += 1; - } - - lfs_crc(&crc, &data, 1); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - - newoff += 1; - } - - crc = lfs_tole32(crc); - err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4); - crc = lfs_fromle32(crc); + c.crc = lfs_tole32(c.crc); + err = lfs_bd_prog(lfs, dir->pair[0], c.off, &c.crc, 4); + c.crc = lfs_fromle32(c.crc); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -613,12 +630,13 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (ncrc != crc) { + if (ncrc != c.crc) { goto relocate; } } break; + relocate: //commit was corrupted LFS_DEBUG("Bad block at %d", dir->pair[0]); @@ -709,14 +727,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { - lfs_off_t oldoff = entry->off; + lfs_off_t oldoff = entry->off; // <- TODO rm me? lfs_ssize_t diff = 0; - lfs_size_t len = 0; - struct lfs_region **tail = ®ions; for (struct lfs_region *r = regions; r; r = r->next) { diff += r->diff; - len += r->len; - tail = &r->next; } // do we still fit? @@ -731,45 +745,44 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } } else { lfs_dir_t olddir = *dir; + lfs_off_t oldoff = entry->off; + lfs_size_t oldsize = lfs_entry_size(entry) - diff; // mark as moving entry->d.type |= LFS_STRUCT_MOVED; int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - LFS_FROM_MEM, oldoff, 0, - {.m.data = &entry->d.type}, 1}); + oldoff, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry->d.type, 1}}); if (err) { return err; } // append updated entry entry->d.type &= LFS_STRUCT_MOVED; - lfs_size_t prefix = regions->off; - lfs_size_t suffix = lfs_entry_size(entry) - len - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off = 0; - r->diff = r->len; + err = lfs_dir_append(lfs, dir, entry, + &(struct lfs_region){ + 0, +lfs_entry_size(entry), + lfs_commit_disk, &(struct lfs_commit_disk){ + olddir.pair[0], entry->off, oldsize, regions}}); + if (err) { + return err; } - regions = &(struct lfs_region){ - LFS_FROM_DISK, 0, +prefix, - {.d.block = olddir.pair[0], .d.off = entry->off}, - prefix, regions}; - *tail = &(struct lfs_region){ - LFS_FROM_DISK, 0, +suffix, - {.d.block = olddir.pair[0], .d.off = entry->off+prefix+len}, - suffix}; - // remove old entry err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - LFS_FROM_MEM, oldoff, -(lfs_entry_size(entry) - diff), - {.m.data = NULL}, 0}); + oldoff, -oldsize, + lfs_commit_mem, &(struct lfs_commit_mem){ + NULL, 0}}); if (err) { return err; } } + // TODO move to dir_commit? + // TODO this doesn't work... // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { @@ -812,9 +825,11 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } // shift out the entry - int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - LFS_FROM_MEM, entry->off, -lfs_entry_size(entry), - {.m.data = NULL}, 0}); + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + entry->off, -lfs_entry_size(entry), + lfs_commit_mem, &(struct lfs_commit_mem){ + NULL, 0}}); if (err) { return err; } @@ -1036,11 +1051,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(entry.d), - {.m.data = &entry.d}, sizeof(entry.d), + 0, +sizeof(entry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +entry.d.nlen, - {.m.data = path}, entry.d.nlen}}); + 0, +entry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + path, entry.d.nlen}}}); if (err) { return err; } @@ -1426,11 +1443,13 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.u.file.size = 0; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(entry.d), - {.m.data = &entry.d}, sizeof(entry.d), + 0, +sizeof(entry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +entry.d.nlen, - {.m.data = path}, entry.d.nlen}}); + 0, +entry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + path, entry.d.nlen}}}); if (err) { return err; } @@ -1648,8 +1667,9 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } @@ -2073,8 +2093,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &oldentry.d}, sizeof(oldentry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &oldentry.d, sizeof(oldentry.d)}}); if (err) { return err; } @@ -2093,22 +2114,26 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { if (prevexists) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &newentry.d}, sizeof(newentry.d), + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &newentry.d, sizeof(newentry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(newentry.d), 0, - {.m.data = newpath}, newentry.d.nlen}}); + sizeof(newentry.d), 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + newpath, newentry.d.nlen}}}); if (err) { return err; } } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - LFS_FROM_MEM, 0, +sizeof(newentry.d), - {.m.data = &newentry.d}, sizeof(newentry.d), + 0, +sizeof(newentry.d), + lfs_commit_mem, &(struct lfs_commit_mem){ + &newentry.d, sizeof(newentry.d)}, &(struct lfs_region){ - LFS_FROM_MEM, 0, +newentry.d.nlen, - {.m.data = newpath}, newentry.d.nlen}}); + 0, +newentry.d.nlen, + lfs_commit_mem, &(struct lfs_commit_mem){ + newpath, newentry.d.nlen}}}); if (err) { return err; } @@ -2275,8 +2300,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { bool valid = false; for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - LFS_FROM_MEM, sizeof(superdir.d), 0, - {.m.data = &superblock.d}, sizeof(superblock.d)}); + sizeof(superdir.d), 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &superblock.d, sizeof(superblock.d)}}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2542,8 +2568,9 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } @@ -2669,8 +2696,9 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - LFS_FROM_MEM, 0, 0, - {.m.data = &entry.d}, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &(struct lfs_commit_mem){ + &entry.d, sizeof(entry.d)}}); if (err) { return err; } From e4a0cd942d05778379b63f4c2c5c417d3cac4f74 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 11 Mar 2018 22:43:06 -0500 Subject: [PATCH 08/31] Take advantage of empty space early in dir search Before, when appending new entries to a directory, we try to find empty space in the last block of a directory chain. This has a nice side-effect that the order of directory entries is maintained. However, this isn't strictly necessary. We're already scanning the directory chain in order, so other than changes to directory order, there's no downside to taking advantage of any free space we come across. --- lfs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lfs.c b/lfs.c index 35bb1646..5563b951 100644 --- a/lfs.c +++ b/lfs.c @@ -683,7 +683,8 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { + if ((0x7fffffff & dir->d.size) + lfs_entry_size(entry) + <= lfs->cfg->block_size) { entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { r->off += entry->off; From 362b0bbe45b803c38838b69c6a922476dc0957cd Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 12 Mar 2018 19:44:07 -0500 Subject: [PATCH 09/31] Minor improvement to from-memory commits Tweaked the commit callback to pass the arguments for from-memory commits explicitly, with non-from-memory commits still being able to hijack the opaque data pointer for additional state. The from-memory commits make up the vast majority of commits in littlefs, so this small change has a noticable impact. --- lfs.c | 88 ++++++++++++++++++++++++----------------------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/lfs.c b/lfs.c index 5563b951..ba207102 100644 --- a/lfs.c +++ b/lfs.c @@ -490,7 +490,8 @@ struct lfs_commit { lfs_off_t off; }; -static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_size_t size) { +static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size) { lfs_crc(&c->crc, data, size); int err = lfs_bd_prog(lfs, c->block, c->off, data, size); c->off += size; @@ -500,46 +501,44 @@ static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, const void *data, lfs_si struct lfs_region { lfs_off_t off; lfs_ssize_t diff; - int (*commit)(lfs_t *lfs, struct lfs_commit *c, const void *p); - const void *data; - struct lfs_region *next; -}; -struct lfs_commit_mem { + int (*commit)(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size); const void *data; lfs_size_t size; + struct lfs_region *next; }; -static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, const void *p) { - const struct lfs_commit_mem *m = p; - return lfs_commit(lfs, c, m->data, m->size); +static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, + const void *data, lfs_size_t size) { + return lfs_commit(lfs, c, data, size); } struct lfs_commit_disk { lfs_block_t block; lfs_off_t off; - lfs_size_t size; struct lfs_region *regions; }; -static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, const void *p) { - const struct lfs_commit_disk *u = p; +static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, + const void *p, lfs_size_t size) { + const struct lfs_commit_disk *d = p; - struct lfs_region *r = u->regions; + struct lfs_region *r = d->regions; lfs_off_t off = 0; while (true) { if (r && r->off == off) { lfs_off_t orig = c->off; - int err = r->commit(lfs, c, r->data); + int err = r->commit(lfs, c, r->data, r->size); if (err) { return err; } off += (c->off - orig) - r->diff; r = r->next; - } else if (off < u->size) { + } else if (off < size) { uint8_t data; - int err = lfs_bd_read(lfs, u->block, u->off + off, &data, 1); + int err = lfs_bd_read(lfs, d->block, d->off + off, &data, 1); if (err) { return err; } @@ -590,12 +589,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, lfs_dir_tole32(&dir->d); err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ - oldpair[1], 0, oldsize, + oldpair[1], 0, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &dir->d, sizeof(dir->d)}, - regions}}); + lfs_commit_mem, &dir->d, sizeof(dir->d), + regions}}, oldsize); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -754,8 +752,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ oldoff, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry->d.type, 1}}); + lfs_commit_mem, &entry->d.type, 1}); if (err) { return err; } @@ -766,7 +763,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, &(struct lfs_region){ 0, +lfs_entry_size(entry), lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, oldsize, regions}}); + olddir.pair[0], entry->off, regions}, oldsize}); if (err) { return err; } @@ -775,8 +772,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ oldoff, -oldsize, - lfs_commit_mem, &(struct lfs_commit_mem){ - NULL, 0}}); + lfs_commit_mem, NULL, 0}); if (err) { return err; } @@ -829,8 +825,7 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ entry->off, -lfs_entry_size(entry), - lfs_commit_mem, &(struct lfs_commit_mem){ - NULL, 0}}); + lfs_commit_mem, NULL, 0}); if (err) { return err; } @@ -1053,12 +1048,10 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}, + lfs_commit_mem, &entry.d, sizeof(entry.d), &(struct lfs_region){ 0, +entry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - path, entry.d.nlen}}}); + lfs_commit_mem, path, entry.d.nlen}}); if (err) { return err; } @@ -1445,12 +1438,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}, + lfs_commit_mem, &entry.d, sizeof(entry.d), &(struct lfs_region){ 0, +entry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - path, entry.d.nlen}}}); + lfs_commit_mem, path, entry.d.nlen}}); if (err) { return err; } @@ -1669,8 +1660,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } @@ -2095,8 +2085,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &oldentry.d, sizeof(oldentry.d)}}); + lfs_commit_mem, &oldentry.d, sizeof(oldentry.d)}); if (err) { return err; } @@ -2116,12 +2105,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &newentry.d, sizeof(newentry.d)}, + lfs_commit_mem, &newentry.d, sizeof(newentry.d), &(struct lfs_region){ sizeof(newentry.d), 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - newpath, newentry.d.nlen}}}); + lfs_commit_mem, newpath, newentry.d.nlen}}); if (err) { return err; } @@ -2129,12 +2116,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ 0, +sizeof(newentry.d), - lfs_commit_mem, &(struct lfs_commit_mem){ - &newentry.d, sizeof(newentry.d)}, + lfs_commit_mem, &newentry.d, sizeof(newentry.d), &(struct lfs_region){ 0, +newentry.d.nlen, - lfs_commit_mem, &(struct lfs_commit_mem){ - newpath, newentry.d.nlen}}}); + lfs_commit_mem, newpath, newentry.d.nlen}}); if (err) { return err; } @@ -2302,8 +2287,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ sizeof(superdir.d), 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &superblock.d, sizeof(superblock.d)}}); + lfs_commit_mem, &superblock.d, sizeof(superblock.d)}); if (err && err != LFS_ERR_CORRUPT) { return err; } @@ -2570,8 +2554,7 @@ static int lfs_relocate(lfs_t *lfs, int err = lfs_dir_update(lfs, &parent, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } @@ -2698,8 +2681,7 @@ int lfs_deorphan(lfs_t *lfs) { err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &(struct lfs_commit_mem){ - &entry.d, sizeof(entry.d)}}); + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } From 03b262b1e8f4fbaff9d3a2bc25e5c42aa7416c86 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 20:32:00 -0500 Subject: [PATCH 10/31] Separated out version of dir remove/append for non-entries This allows updates to directories without needing to allocate an entry struct for every call. --- lfs.c | 177 ++++++++++++++++++++++++++++++---------------------------- 1 file changed, 91 insertions(+), 86 deletions(-) diff --git a/lfs.c b/lfs.c index ba207102..313198b9 100644 --- a/lfs.c +++ b/lfs.c @@ -677,15 +677,14 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { +static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, + lfs_off_t *off, lfs_size_t size, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if ((0x7fffffff & dir->d.size) + lfs_entry_size(entry) - <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; + if ((0x7fffffff & dir->d.size) + size <= lfs->cfg->block_size) { + *off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + r->off += *off; } return lfs_dir_commit(lfs, dir, regions); @@ -701,9 +700,9 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; + *off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + r->off += *off; } err = lfs_dir_commit(lfs, dir, regions); @@ -724,9 +723,70 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, } } +static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, + lfs_off_t off, lfs_size_t size) { + // check if we should just drop the directory block + if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + size) { + lfs_dir_t pdir; + int res = lfs_pred(lfs, dir->pair, &pdir); + if (res < 0) { + return res; + } + + if (pdir.d.size & 0x80000000) { + pdir.d.size &= dir->d.size | 0x7fffffff; + pdir.d.tail[0] = dir->d.tail[0]; + pdir.d.tail[1] = dir->d.tail[1]; + return lfs_dir_commit(lfs, &pdir, NULL); + } + } + + // shift out the entry + int err = lfs_dir_commit(lfs, dir, + &(struct lfs_region){ + off, -size, + lfs_commit_mem, NULL, 0}); + if (err) { + return err; + } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff == off) { + f->pair[0] = 0xffffffff; + f->pair[1] = 0xffffffff; + } else if (f->poff > off) { + f->poff -= size; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > off) { + d->off -= size; + d->pos -= size; + } + } + } + + return 0; +} + +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { + return lfs_dir_append_(lfs, dir, + &entry->off, lfs_entry_size(entry), regions); +} + +static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { + return lfs_dir_remove_(lfs, dir, + entry->off, lfs_entry_size(entry)); +} + static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { - lfs_off_t oldoff = entry->off; // <- TODO rm me? lfs_ssize_t diff = 0; for (struct lfs_region *r = regions; r; r = r->next) { diff += r->diff; @@ -742,6 +802,24 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, if (err) { return err; } + + // shift over any files/directories that are affected + for (lfs_file_t *f = lfs->files; f; f = f->next) { + if (lfs_paircmp(f->pair, dir->pair) == 0) { + if (f->poff > entry->off) { + f->poff += diff; + } + } + } + + for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { + if (lfs_paircmp(d->pair, dir->pair) == 0) { + if (d->off > entry->off) { + d->off += diff; + d->pos += diff; + } + } + } } else { lfs_dir_t olddir = *dir; lfs_off_t oldoff = entry->off; @@ -756,9 +834,9 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, if (err) { return err; } + entry->d.type &= LFS_STRUCT_MOVED; // append updated entry - entry->d.type &= LFS_STRUCT_MOVED; err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ 0, +lfs_entry_size(entry), @@ -769,86 +847,13 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_commit(lfs, &olddir, - &(struct lfs_region){ - oldoff, -oldsize, - lfs_commit_mem, NULL, 0}); + err = lfs_dir_remove_(lfs, dir, oldoff, oldsize); if (err) { return err; } - } - // TODO move to dir_commit? - // TODO this doesn't work... - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == oldoff) { - f->poff = entry->off; - } else if (f->poff > entry->off) { - f->poff += diff; - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off += diff; - d->pos += diff; - } - } - } - - return 0; -} - -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 - + lfs_entry_size(entry)) { - lfs_dir_t pdir; - int res = lfs_pred(lfs, dir->pair, &pdir); - if (res < 0) { - return res; - } - - if (pdir.d.size & 0x80000000) { - pdir.d.size &= dir->d.size | 0x7fffffff; - pdir.d.tail[0] = dir->d.tail[0]; - pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL); - } - } - - // shift out the entry - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - entry->off, -lfs_entry_size(entry), - lfs_commit_mem, NULL, 0}); - if (err) { - return err; - } - - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off) { - f->pair[0] = 0xffffffff; - f->pair[1] = 0xffffffff; - } else if (f->poff > entry->off) { - f->poff -= lfs_entry_size(entry); - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off -= lfs_entry_size(entry); - d->pos -= lfs_entry_size(entry); - } - } + // TODO remove file under file? + // TODO need to shift entries? } return 0; From 9273ac708b69a1eb53b1add1058af3e5526986a8 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 20:58:29 -0500 Subject: [PATCH 11/31] Added size field to entry structure The size field is redundant, since an entry's size can be determined from the nlen+elen+alen+4. However, as you may have guessed from that expression, calculating the size this way is a bit roundabout and inefficient. Despite its redundancy, it's cheaper to store the size in the entry, though with a minor RAM cost. Note, extra care must now be taken to make sure these size and len fields don't fall out of sync. --- lfs.c | 82 +++++++++++++++++++++++------------------------------------ lfs.h | 3 +-- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/lfs.c b/lfs.c index 313198b9..66c8163c 100644 --- a/lfs.c +++ b/lfs.c @@ -397,10 +397,6 @@ static inline bool lfs_pairsync( (paira[0] == pairb[1] && paira[1] == pairb[0]); } -static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { - return 4 + entry->d.elen + entry->d.alen + entry->d.nlen; -} - static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { // allocate pair of dir blocks for (int i = 0; i < 2; i++) { @@ -677,14 +673,14 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, - lfs_off_t *off, lfs_size_t size, struct lfs_region *regions) { +static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, + lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on while (true) { - if ((0x7fffffff & dir->d.size) + size <= lfs->cfg->block_size) { - *off = dir->d.size - 4; + if ((0x7fffffff & dir->d.size) + entry->size <= lfs->cfg->block_size) { + entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += *off; + r->off += entry->off; } return lfs_dir_commit(lfs, dir, regions); @@ -700,9 +696,9 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, dir->d.tail[0] = olddir.d.tail[0]; dir->d.tail[1] = olddir.d.tail[1]; - *off = dir->d.size - 4; + entry->off = dir->d.size - 4; for (struct lfs_region *r = regions; r; r = r->next) { - r->off += *off; + r->off += entry->off; } err = lfs_dir_commit(lfs, dir, regions); @@ -723,10 +719,10 @@ static int lfs_dir_append_(lfs_t *lfs, lfs_dir_t *dir, } } -static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, - lfs_off_t off, lfs_size_t size) { +static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, + const lfs_entry_t *entry) { // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + size) { + if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + entry->size) { lfs_dir_t pdir; int res = lfs_pred(lfs, dir->pair, &pdir); if (res < 0) { @@ -744,7 +740,7 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, // shift out the entry int err = lfs_dir_commit(lfs, dir, &(struct lfs_region){ - off, -size, + entry->off, -entry->size, lfs_commit_mem, NULL, 0}); if (err) { return err; @@ -753,20 +749,20 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == off) { + if (f->poff == entry->off) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; - } else if (f->poff > off) { - f->poff -= size; + } else if (f->poff > entry->off) { + f->poff -= entry->size; } } } for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > off) { - d->off -= size; - d->pos -= size; + if (d->off > entry->off) { + d->off -= entry->size; + d->pos -= entry->size; } } } @@ -774,17 +770,6 @@ static int lfs_dir_remove_(lfs_t *lfs, lfs_dir_t *dir, return 0; } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - return lfs_dir_append_(lfs, dir, - &entry->off, lfs_entry_size(entry), regions); -} - -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - return lfs_dir_remove_(lfs, dir, - entry->off, lfs_entry_size(entry)); -} - static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { lfs_ssize_t diff = 0; @@ -823,7 +808,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } else { lfs_dir_t olddir = *dir; lfs_off_t oldoff = entry->off; - lfs_size_t oldsize = lfs_entry_size(entry) - diff; + lfs_size_t oldsize = entry->size - diff; // mark as moving entry->d.type |= LFS_STRUCT_MOVED; @@ -839,7 +824,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, // append updated entry err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ - 0, +lfs_entry_size(entry), + 0, +entry->size, lfs_commit_disk, &(struct lfs_commit_disk){ olddir.pair[0], entry->off, regions}, oldsize}); if (err) { @@ -847,13 +832,10 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_remove_(lfs, dir, oldoff, oldsize); + err = lfs_dir_remove(lfs, dir, &(lfs_entry_t){oldoff, oldsize}); if (err) { return err; } - - // TODO remove file under file? - // TODO need to shift entries? } return 0; @@ -883,8 +865,9 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } entry->off = dir->off; - dir->off += lfs_entry_size(entry); - dir->pos += lfs_entry_size(entry); + entry->size = 4 + entry->d.elen + entry->d.alen + entry->d.nlen; + dir->off += entry->size; + dir->pos += entry->size; return 0; } @@ -903,9 +886,6 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, if (pathname[0] == '\0') { *entry = (lfs_entry_t){ .d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR, - .d.elen = sizeof(entry->d) - 4, - .d.alen = 0, - .d.nlen = 0, .d.u.dir[0] = lfs->root[0], .d.u.dir[1] = lfs->root[1], }; @@ -960,7 +940,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } int res = lfs_bd_cmp(lfs, dir->pair[0], - entry->off + 4+entry->d.elen+entry->d.alen, + entry->off + entry->size - pathlen, pathname, pathlen); if (res < 0) { return res; @@ -1046,6 +1026,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.nlen = strlen(path); entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; @@ -1164,7 +1145,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } int err = lfs_bd_read(lfs, dir->pair[0], - entry.off + 4+entry.d.elen+entry.d.alen, + entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { return err; @@ -1440,6 +1421,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.nlen = strlen(path); entry.d.u.file.head = 0xffffffff; entry.d.u.file.size = 0; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ 0, +sizeof(entry.d), @@ -1959,7 +1942,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { strcpy(info->name, "/"); } else { err = lfs_bd_read(lfs, cwd.pair[0], - entry.off + 4+entry.d.elen+entry.d.alen, + entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { return err; @@ -2272,7 +2255,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { // write superblocks lfs_superblock_t superblock = { - .off = sizeof(superdir.d), .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, .d.nlen = sizeof(superblock.d.magic), @@ -2374,8 +2356,6 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } // iterate over metadata pairs - lfs_dir_t dir; - lfs_entry_t entry; lfs_block_t cwd[2] = {0, 1}; while (true) { @@ -2386,12 +2366,14 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } + lfs_dir_t dir; int err = lfs_dir_fetch(lfs, &dir, cwd); if (err) { return err; } // iterate over contents + lfs_entry_t entry; while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { err = lfs_bd_read(lfs, dir.pair[0], dir.off, &entry.d, sizeof(entry.d)); @@ -2400,7 +2382,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return err; } - dir.off += lfs_entry_size(&entry); + dir.off += 4 + entry.d.elen + entry.d.alen + entry.d.nlen; if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); diff --git a/lfs.h b/lfs.h index 4a12c4a2..b35ad249 100644 --- a/lfs.h +++ b/lfs.h @@ -196,6 +196,7 @@ struct lfs_info { /// littlefs data structures /// typedef struct lfs_entry { lfs_off_t off; + lfs_size_t size; struct lfs_disk_entry { uint8_t type; @@ -249,8 +250,6 @@ typedef struct lfs_dir { } lfs_dir_t; typedef struct lfs_superblock { - lfs_off_t off; - struct lfs_disk_superblock { uint8_t type; uint8_t elen; From fb2304487206c6deed88f941da45c3e6113ad5f7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 15 Mar 2018 21:26:03 -0500 Subject: [PATCH 12/31] Fixed big-endian support for entry structures --- lfs.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lfs.c b/lfs.c index 66c8163c..cfbc2b91 100644 --- a/lfs.c +++ b/lfs.c @@ -349,11 +349,10 @@ static void lfs_entry_fromle32(struct lfs_disk_entry *d) { d->u.dir[1] = lfs_fromle32(d->u.dir[1]); } -// TODO -//static void lfs_entry_tole32(struct lfs_disk_entry *d) { -// d->u.dir[0] = lfs_tole32(d->u.dir[0]); -// d->u.dir[1] = lfs_tole32(d->u.dir[1]); -//} +static void lfs_entry_tole32(struct lfs_disk_entry *d) { + d->u.dir[0] = lfs_tole32(d->u.dir[0]); + d->u.dir[1] = lfs_tole32(d->u.dir[1]); +} static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->root[0] = lfs_fromle32(d->root[0]); @@ -683,7 +682,10 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } - return lfs_dir_commit(lfs, dir, regions); + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); + return err; } // we need to allocate a new dir block @@ -701,7 +703,9 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } + lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -783,7 +787,9 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, r->off += entry->off; } + lfs_entry_tole32(&entry->d); int err = lfs_dir_commit(lfs, dir, regions); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -807,32 +813,35 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } } else { lfs_dir_t olddir = *dir; - lfs_off_t oldoff = entry->off; - lfs_size_t oldsize = entry->size - diff; + lfs_entry_t oldentry = { + .off = entry->off, + .size = entry->size - diff, + .d.type = entry->d.type | LFS_STRUCT_MOVED, + }; // mark as moving - entry->d.type |= LFS_STRUCT_MOVED; int err = lfs_dir_commit(lfs, &olddir, &(struct lfs_region){ - oldoff, 0, - lfs_commit_mem, &entry->d.type, 1}); + oldentry.off, 0, + lfs_commit_mem, &oldentry.d.type, 1}); if (err) { return err; } - entry->d.type &= LFS_STRUCT_MOVED; // append updated entry + lfs_entry_tole32(&entry->d); err = lfs_dir_append(lfs, dir, entry, &(struct lfs_region){ 0, +entry->size, lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, regions}, oldsize}); + olddir.pair[0], entry->off, regions}, oldentry.size}); + lfs_entry_fromle32(&entry->d); if (err) { return err; } // remove old entry - err = lfs_dir_remove(lfs, dir, &(lfs_entry_t){oldoff, oldsize}); + err = lfs_dir_remove(lfs, dir, &oldentry); if (err) { return err; } @@ -1645,10 +1654,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + lfs_entry_tole32(&entry.d); err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ 0, 0, lfs_commit_mem, &entry.d, sizeof(entry.d)}); + lfs_entry_fromle32(&entry.d); if (err) { return err; } @@ -2073,7 +2084,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { err = lfs_dir_update(lfs, &oldcwd, &oldentry, &(struct lfs_region){ 0, 0, - lfs_commit_mem, &oldentry.d, sizeof(oldentry.d)}); + lfs_commit_mem, &oldentry.d.type, 1}); if (err) { return err; } From 836e23895a0e9205e6b530112dcc9374306417d7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Mar 2018 10:28:14 -0500 Subject: [PATCH 13/31] Shoehorned in hacky implementation of inline files Proof-of-concept implementation of inline files that stores the file's content directly in its parent's directory pair. Inline files are indicated by a different type stored in an entry's struct field, and take advantage of resizable entries. Where a normal file's entry would normally hold the reference to the CTZ skip-list, an inline file's entry contains the contents of the actual file. Unfortunately, storing the inline file on disk is the easy part. We also need to manage inline files in the internals of littlefs and provide the same operations that we do on normal files, all while reusing as much code as possible to avoid a significant increase in code cost. There is a relatively simple, though maybe a bit hacky, solution here. If a file fits entirely in a cache line, the file logic never actually has to go to disk. This means we can just give the file a "pretend" block (hopefully one that would assert if ever written to), and carry out file operations as normal, as long as we catch the file before it exceeds the cache line and write out the file to an actual disk. --- lfs.c | 163 ++++++++++++++++++++++++++++++++------------- lfs.h | 28 +++++--- tests/test_dirs.sh | 2 +- 3 files changed, 133 insertions(+), 60 deletions(-) diff --git a/lfs.c b/lfs.c index cfbc2b91..d0128636 100644 --- a/lfs.c +++ b/lfs.c @@ -24,7 +24,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (pcache && block == pcache->block && off >= pcache->off && @@ -68,6 +68,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, } // load to cache, first condition can no longer fail + LFS_ASSERT(block < lfs->cfg->block_count); rcache->block = block; rcache->off = off - (off % lfs->cfg->read_size); int err = lfs->cfg->read(lfs->cfg, rcache->block, @@ -121,6 +122,7 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, static int lfs_cache_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache) { if (pcache->block != 0xffffffff) { + LFS_ASSERT(pcache->block < lfs->cfg->block_count); int err = lfs->cfg->prog(lfs->cfg, pcache->block, pcache->off, pcache->buffer, lfs->cfg->prog_size); if (err) { @@ -149,7 +151,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -181,6 +183,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, if (off % lfs->cfg->prog_size == 0 && size >= lfs->cfg->prog_size) { // bypass pcache? + LFS_ASSERT(block < lfs->cfg->block_count); lfs_size_t diff = size - (size % lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); if (err) { @@ -240,6 +243,7 @@ static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, } static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + LFS_ASSERT(block < lfs->cfg->block_count); return lfs->cfg->erase(lfs->cfg, block); } @@ -851,7 +855,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + while (dir->off >= (0x7fffffff & dir->d.size)-4) { if (!(0x80000000 & dir->d.size)) { entry->off = dir->off; return LFS_ERR_NOENT; @@ -942,8 +946,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || + if (((0xf & entry->d.type) != LFS_TYPE_REG && + (0xf & entry->d.type) != LFS_TYPE_DIR) || entry->d.nlen != pathlen) { continue; } @@ -1126,8 +1130,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return (err == LFS_ERR_NOENT) ? 0 : err; } - if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { + if ((0xf & entry.d.type) != LFS_TYPE_REG && + (0xf & entry.d.type) != LFS_TYPE_DIR) { continue; } @@ -1149,8 +1153,10 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } info->type = 0xf & entry.d.type; - if (info->type == LFS_TYPE_REG) { + if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; + } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = entry.d.elen; } int err = lfs_bd_read(lfs, dir->pair[0], @@ -1424,18 +1430,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // create entry to remember name - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d) - 4; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = 0; entry.d.alen = 0; entry.d.nlen = strlen(path); - entry.d.u.file.head = 0xffffffff; - entry.d.u.file.size = 0; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - 0, +sizeof(entry.d), - lfs_commit_mem, &entry.d, sizeof(entry.d), + 0, +4, + lfs_commit_mem, &entry.d, 4, &(struct lfs_region){ 0, +entry.d.nlen, lfs_commit_mem, path, entry.d.nlen}}); @@ -1481,6 +1485,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } + // load inline files + if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + file->head = 0xfffffffe; + file->size = entry.d.elen; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; + err = lfs_bd_read(lfs, cwd.pair[0], + entry.off + 4, + file->cache.buffer, file->size); + if (err) { + lfs_free(file->cache.buffer); + return err; + } + } + // add to list of files file->next = lfs->files; lfs->files = file; @@ -1556,6 +1576,20 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_INLINE) { + // do nothing since we won't need the cache for anything else + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } + + if (file->flags & LFS_F_WRITING) { + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + } + + return 0; + } + if (file->flags & LFS_F_READING) { // just drop read cache file->cache.block = 0xffffffff; @@ -1642,6 +1676,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } + // TODO entry read? lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); @@ -1649,19 +1684,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (err) { return err; } + LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; + if (file->flags & LFS_F_INLINE) { + file->size = lfs_max(file->pos, file->size); + lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.elen = file->size; - lfs_entry_tole32(&entry.d); - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); - lfs_entry_fromle32(&entry.d); - if (err) { - return err; + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, 4, + &(struct lfs_region){ + 4, diff, + lfs_commit_mem, file->cache.buffer, file->size}}); + if (err) { + return err; + } + } else { + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; + + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, sizeof(entry.d)}); + if (err) { + return err; + } } file->flags &= ~LFS_F_DIRTY; @@ -1699,11 +1750,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos, &file->block, &file->off); - if (err) { - return err; + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } } file->flags |= LFS_F_READING; @@ -1761,31 +1817,42 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } while (nsize > 0) { + //printf("pos %d\n", file->pos + nsize); + // TODO combine with block allocation? + if (file->pos + nsize >= LFS_INLINE_MAX) { + file->flags &= ~LFS_F_INLINE; + } + // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { - if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { - // find out which block we're extending from - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos-1, &file->block, &file->off); + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { + // find out which block we're extending from + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos-1, &file->block, &file->off); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + // mark cache as dirty since we may have read data into it + file->cache.block = 0xffffffff; + } + + // extend file with new blocks + lfs_alloc_ack(lfs); + int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, + file->block, file->pos, + &file->block, &file->off); if (err) { file->flags |= LFS_F_ERRED; return err; } - - // mark cache as dirty since we may have read data into it - file->cache.block = 0xffffffff; - } - - // extend file with new blocks - lfs_alloc_ack(lfs); - int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, - file->block, file->pos, - &file->block, &file->off); - if (err) { - file->flags |= LFS_F_ERRED; - return err; } file->flags |= LFS_F_WRITING; diff --git a/lfs.h b/lfs.h index b35ad249..75033b4c 100644 --- a/lfs.h +++ b/lfs.h @@ -55,6 +55,10 @@ typedef uint32_t lfs_block_t; #define LFS_NAME_MAX 255 #endif +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 255 +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -82,25 +86,27 @@ enum lfs_type { // on disk structure LFS_STRUCT_CTZ = 0x10, LFS_STRUCT_DIR = 0x20, + LFS_STRUCT_INLINE = 0x30, LFS_STRUCT_MOVED = 0x80, }; // File open flags enum lfs_open_flags { // open flags - LFS_O_RDONLY = 1, // Open a file as read only - LFS_O_WRONLY = 2, // Open a file as write only - LFS_O_RDWR = 3, // Open a file as read and write - LFS_O_CREAT = 0x0100, // Create a file if it does not exist - LFS_O_EXCL = 0x0200, // Fail if a file already exists - LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size - LFS_O_APPEND = 0x0800, // Move to end of file on every write + LFS_O_RDONLY = 1, // Open a file as read only + LFS_O_WRONLY = 2, // Open a file as write only + LFS_O_RDWR = 3, // Open a file as read and write + LFS_O_CREAT = 0x0100, // Create a file if it does not exist + LFS_O_EXCL = 0x0200, // Fail if a file already exists + LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size + LFS_O_APPEND = 0x0800, // Move to end of file on every write // internally used flags - LFS_F_DIRTY = 0x10000, // File does not match storage - LFS_F_WRITING = 0x20000, // File has been written since last flush - LFS_F_READING = 0x40000, // File has been read since last flush - LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_DIRTY = 0x10000, // File does not match storage + LFS_F_WRITING = 0x20000, // File has been written since last flush + LFS_F_READING = 0x40000, // File has been read since last flush + LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_INLINE = 0x100000, // Currently inlined in directory entry }; // File seek flags diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7a..11dc26ef 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -357,7 +357,7 @@ tests/test.py << TEST TEST echo "--- Multi-block directory with files ---" -tests/test.py << TEST +tests/test.py -s << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) { From d8cadecba63c41c71a6a1796a25a197e8b07020f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Mar 2018 20:32:16 -0500 Subject: [PATCH 14/31] Better implementation of inline files, now with overflowing Now when a file overflows the max inline file size, it will be correctly written out to a proper block. Additionally, tweaked corner cases around inline file, however this still needs significant testing. A real neat part that surprised me is that littlefs _already_ contains the logic for writing out inline files: in lfs_file_relocate! With a bit of tweaking, littlefs can pull off both the overflow from inline to normal files _and_ the relocating of bad blocks in files with the same piece of logic. --- lfs.c | 91 ++++++++++++++++++++++++++++++---------------- tests/test_dirs.sh | 2 +- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/lfs.c b/lfs.c index d0128636..56559ccd 100644 --- a/lfs.c +++ b/lfs.c @@ -554,6 +554,7 @@ static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, } } +// TODO handle overflowing reads (zero?) static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, struct lfs_region *regions) { // state for copying over @@ -1152,6 +1153,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } + // TODO common info constructor? + // TODO also used in lfs_stat info->type = 0xf & entry.d.type; if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; @@ -1452,23 +1455,6 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_EXIST; } - // setup file struct - file->pair[0] = cwd.pair[0]; - file->pair[1] = cwd.pair[1]; - file->poff = entry.off; - file->head = entry.d.u.file.head; - file->size = entry.d.u.file.size; - file->flags = flags; - file->pos = 0; - - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - file->head = 0xffffffff; - file->size = 0; - } - // allocate buffer if needed file->cache.block = 0xffffffff; if (lfs->cfg->file_buffer) { @@ -1485,6 +1471,25 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } + // TODO combine these below? + // setup file struct + file->pair[0] = cwd.pair[0]; + file->pair[1] = cwd.pair[1]; + file->poff = entry.off; + file->head = entry.d.u.file.head; + file->size = entry.d.u.file.size; + file->flags = flags; + file->pos = 0; + + if (flags & LFS_O_TRUNC) { + if (file->size != 0) { + file->flags |= LFS_F_DIRTY; + } + + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = 0; + } + // load inline files if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { file->head = 0xfffffffe; @@ -1528,9 +1533,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { -relocate: - LFS_DEBUG("Bad block at %d", file->block); - +relocate:; // just relocate what exists into new block lfs_block_t nblock; int err = lfs_alloc(lfs, &nblock); @@ -1583,6 +1586,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { } if (file->flags & LFS_F_WRITING) { + file->size = lfs_max(file->pos, file->size); file->flags &= ~LFS_F_WRITING; file->flags |= LFS_F_DIRTY; } @@ -1642,6 +1646,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { break; relocate: + LFS_DEBUG("Bad block at %d", file->block); err = lfs_file_relocate(lfs, file); if (err) { return err; @@ -1676,7 +1681,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - // TODO entry read? + // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); @@ -1685,12 +1690,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; if (file->flags & LFS_F_INLINE) { - file->size = lfs_max(file->pos, file->size); lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = file->size; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ @@ -1703,12 +1708,17 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } } else { + lfs_ssize_t diff = sizeof(entry.d) - entry.d.elen; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; + entry.d.elen = sizeof(entry.d); entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // TODO combine up? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, 0, + 0, diff, lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; @@ -1747,12 +1757,13 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, nsize = size; while (nsize > 0) { + // TODO can this be collapsed? // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { if (file->flags & LFS_F_INLINE) { file->block = 0xfffffffe; - file->off = 0; + file->off = file->pos; } else { int err = lfs_ctz_find(lfs, &file->cache, NULL, file->head, file->size, @@ -1816,19 +1827,31 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } } - while (nsize > 0) { - //printf("pos %d\n", file->pos + nsize); - // TODO combine with block allocation? - if (file->pos + nsize >= LFS_INLINE_MAX) { - file->flags &= ~LFS_F_INLINE; + // TODO combine with block allocation? + // TODO need to move out if no longer fits in block also + // TODO store INLINE_MAX in superblock? + if ((file->pos + nsize >= LFS_INLINE_MAX) || + (file->pos + nsize >= lfs->cfg->read_size)) { + int err = lfs_file_relocate(lfs, file); + if (err) { + file->flags |= LFS_F_ERRED; + return err; } + file->flags &= ~LFS_F_INLINE; + file->flags |= LFS_F_WRITING; + } + + while (nsize > 0) { + // TODO can this be collapsed? + // TODO can we reduce this now that block 0 is never allocated? + // TODO actually, how does this behave if inline max == 0? // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { if (file->flags & LFS_F_INLINE) { file->block = 0xfffffffe; - file->off = 0; + file->off = file->pos; } else { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from @@ -1920,6 +1943,8 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, return file->pos; } +// TODO handle inlining? +// TODO note at least needs tests for trunc on inlined file int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; @@ -2012,8 +2037,10 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { memset(info, 0, sizeof(*info)); info->type = 0xf & entry.d.type; - if (info->type == LFS_TYPE_REG) { + if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; + } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = entry.d.elen; } if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 11dc26ef..53d76f7a 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -357,7 +357,7 @@ tests/test.py << TEST TEST echo "--- Multi-block directory with files ---" -tests/test.py -s << TEST +tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) { From 701e4fa4386924ca34c31f8a1fee7e2497e9494b Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 18 Mar 2018 20:36:48 -0500 Subject: [PATCH 15/31] Fixed a handful of bugs as result of testing --- lfs.c | 211 ++++++++++++++++++++++-------------------- tests/test_alloc.sh | 12 ++- tests/test_corrupt.sh | 2 +- 3 files changed, 123 insertions(+), 102 deletions(-) diff --git a/lfs.c b/lfs.c index 56559ccd..376e083a 100644 --- a/lfs.c +++ b/lfs.c @@ -821,7 +821,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t oldentry = { .off = entry->off, .size = entry->size - diff, - .d.type = entry->d.type | LFS_STRUCT_MOVED, + .d.type = entry->d.type | LFS_STRUCT_MOVED, // TODO FIXME when changing type?? }; // mark as moving @@ -846,7 +846,12 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } // remove old entry - err = lfs_dir_remove(lfs, dir, &oldentry); + err = lfs_dir_fetch(lfs, &olddir, olddir.pair); + if (err) { + return err; + } + + err = lfs_dir_remove(lfs, &olddir, &oldentry); if (err) { return err; } @@ -1579,78 +1584,69 @@ relocate:; } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_INLINE) { - // do nothing since we won't need the cache for anything else - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } - - if (file->flags & LFS_F_WRITING) { - file->size = lfs_max(file->pos, file->size); - file->flags &= ~LFS_F_WRITING; - file->flags |= LFS_F_DIRTY; - } - - return 0; - } - if (file->flags & LFS_F_READING) { - // just drop read cache - file->cache.block = 0xffffffff; + if (!(file->flags & LFS_F_INLINE)) { + // just drop read cache + file->cache.block = 0xffffffff; + } file->flags &= ~LFS_F_READING; } if (file->flags & LFS_F_WRITING) { lfs_off_t pos = file->pos; - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; - lfs->rcache.block = 0xffffffff; + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; + lfs->rcache.block = 0xffffffff; - while (file->pos < file->size) { - // copy over a byte at a time, leave it up to caching - // to make this efficient - uint8_t data; - lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); - if (res < 0) { - return res; - } + while (file->pos < file->size) { + // copy over a byte at a time, leave it up to caching + // to make this efficient + uint8_t data; + lfs_ssize_t res = lfs_file_read(lfs, &orig, &data, 1); + if (res < 0) { + return res; + } - res = lfs_file_write(lfs, file, &data, 1); - if (res < 0) { - return res; - } + res = lfs_file_write(lfs, file, &data, 1); + if (res < 0) { + return res; + } - // keep our reference to the rcache in sync - if (lfs->rcache.block != 0xffffffff) { - orig.cache.block = 0xffffffff; - lfs->rcache.block = 0xffffffff; + // keep our reference to the rcache in sync + if (lfs->rcache.block != 0xffffffff) { + orig.cache.block = 0xffffffff; + lfs->rcache.block = 0xffffffff; + } } - } - // write out what we have - while (true) { - int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // write out what we have + while (true) { + int err = lfs_cache_flush(lfs, &file->cache, &lfs->rcache); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; } - return err; - } - break; + break; relocate: - LFS_DEBUG("Bad block at %d", file->block); - err = lfs_file_relocate(lfs, file); - if (err) { - return err; + LFS_DEBUG("Bad block at %d", file->block); + err = lfs_file_relocate(lfs, file); + if (err) { + return err; + } } + } else { + file->size = lfs_max(file->pos, file->size); } // actual file updates @@ -1691,35 +1687,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - if (file->flags & LFS_F_INLINE) { - lfs_ssize_t diff = file->size - entry.d.elen; - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size; + if (!(file->flags & LFS_F_INLINE)) { + lfs_ssize_t diff = sizeof(entry.d)-4 - entry.d.elen; + entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; + entry.d.elen = sizeof(entry.d)-4; + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // TODO combine down? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 4, diff, - lfs_commit_mem, file->cache.buffer, file->size}}); + 0, diff, + lfs_commit_mem, &entry.d, sizeof(entry.d)}); if (err) { return err; } } else { - lfs_ssize_t diff = sizeof(entry.d) - entry.d.elen; - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d); - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; + lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = file->size; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - // TODO combine up? err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){ - 0, diff, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + 0, 0, + lfs_commit_mem, &entry.d, 4, + &(struct lfs_region){ + 4, diff, + lfs_commit_mem, file->cache.buffer, file->size}}); if (err) { return err; } @@ -1761,16 +1757,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { - if (file->flags & LFS_F_INLINE) { - file->block = 0xfffffffe; - file->off = file->pos; - } else { + if (!(file->flags & LFS_F_INLINE)) { int err = lfs_ctz_find(lfs, &file->cache, NULL, file->head, file->size, file->pos, &file->block, &file->off); if (err) { return err; } + } else { + file->block = 0xfffffffe; + file->off = file->pos; } file->flags |= LFS_F_READING; @@ -1830,8 +1826,14 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO combine with block allocation? // TODO need to move out if no longer fits in block also // TODO store INLINE_MAX in superblock? - if ((file->pos + nsize >= LFS_INLINE_MAX) || - (file->pos + nsize >= lfs->cfg->read_size)) { + // TODO what if inline files is > block size (ie 128) + if ((file->flags & LFS_F_INLINE) && ( + (file->pos + nsize >= LFS_INLINE_MAX) || + (file->pos + nsize >= lfs->cfg->read_size))) { + file->block = 0xfffffffe; + file->off = file->pos; + + lfs_alloc_ack(lfs); int err = lfs_file_relocate(lfs, file); if (err) { file->flags |= LFS_F_ERRED; @@ -1849,10 +1851,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { - if (file->flags & LFS_F_INLINE) { - file->block = 0xfffffffe; - file->off = file->pos; - } else { + if (!(file->flags & LFS_F_INLINE)) { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from int err = lfs_ctz_find(lfs, &file->cache, NULL, @@ -1876,6 +1875,9 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_ERRED; return err; } + } else { + file->block = 0xfffffffe; + file->off = file->pos; } file->flags |= LFS_F_WRITING; @@ -1944,7 +1946,6 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, } // TODO handle inlining? -// TODO note at least needs tests for trunc on inlined file int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; @@ -2179,6 +2180,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { &(struct lfs_region){ 0, 0, lfs_commit_mem, &oldentry.d.type, 1}); + oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; } @@ -2193,26 +2195,39 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); + newentry.size = 4 + newentry.d.elen + newentry.d.alen + newentry.d.nlen; if (prevexists) { err = lfs_dir_update(lfs, &newcwd, &newentry, &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, sizeof(newentry.d), - &(struct lfs_region){ - sizeof(newentry.d), 0, - lfs_commit_mem, newpath, newentry.d.nlen}}); + 0, newentry.size - preventry.size, + lfs_commit_disk, &(struct lfs_commit_disk){ + oldcwd.pair[0], oldentry.off, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &newentry.d, 4, + &(struct lfs_region){ + newentry.size - newentry.d.nlen, + +newentry.d.nlen-oldentry.d.nlen, + lfs_commit_mem, newpath, newentry.d.nlen}}}, + oldentry.size}); if (err) { return err; } } else { err = lfs_dir_append(lfs, &newcwd, &newentry, &(struct lfs_region){ - 0, +sizeof(newentry.d), - lfs_commit_mem, &newentry.d, sizeof(newentry.d), - &(struct lfs_region){ - 0, +newentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}); + 0, +newentry.size, + lfs_commit_disk, &(struct lfs_commit_disk){ + oldcwd.pair[0], oldentry.off, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &newentry.d, 4, + &(struct lfs_region){ + newentry.size - newentry.d.nlen, + +newentry.d.nlen-oldentry.d.nlen, + lfs_commit_mem, newpath, newentry.d.nlen}}}, + oldentry.size}); if (err) { return err; } @@ -2507,7 +2522,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { // iterate over any open files for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (f->flags & LFS_F_DIRTY) { + if ((f->flags & LFS_F_DIRTY) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, f->head, f->size, cb, data); if (err) { @@ -2515,7 +2530,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } - if (f->flags & LFS_F_WRITING) { + if ((f->flags & LFS_F_WRITING) && !(f->flags & LFS_F_INLINE)) { int err = lfs_ctz_traverse(lfs, &lfs->rcache, &f->cache, f->block, f->pos, cb, data); if (err) { diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index 8c814908..d3c8da40 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -274,9 +274,12 @@ TEST tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - // create one block whole for half a directory + // create one block hole for half a directory lfs_file_open(&lfs, &file[0], "bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - lfs_file_write(&lfs, &file[0], (void*)"hi", 2) => 2; + for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + memcpy(&buffer[i], "hi", 2); + } + lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => cfg.block_size; lfs_file_close(&lfs, &file[0]) => 0; lfs_file_open(&lfs, &file[0], "exhaustion", LFS_O_WRONLY | LFS_O_CREAT); @@ -295,7 +298,10 @@ tests/test.py << TEST lfs_mkdir(&lfs, "splitdir") => 0; lfs_file_open(&lfs, &file[0], "splitdir/bump", LFS_O_WRONLY | LFS_O_CREAT) => 0; - lfs_file_write(&lfs, &file[0], buffer, size) => LFS_ERR_NOSPC; + for (lfs_size_t i = 0; i < cfg.block_size; i += 2) { + memcpy(&buffer[i], "hi", 2); + } + lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => LFS_ERR_NOSPC; lfs_file_close(&lfs, &file[0]) => 0; lfs_unmount(&lfs) => 0; diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index 44f1caee..6af984ff 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -88,7 +88,7 @@ do rm -rf blocks mkdir blocks lfs_mktree - chmod a-w blocks/$(printf '%x' $i) + chmod a-w blocks/$(printf '%x' $i) || true lfs_mktree lfs_chktree done From d0e0453651fb054004ac2212d70d49784afef1c0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 Mar 2018 16:11:36 -0500 Subject: [PATCH 16/31] Changed how we write out superblock to use append Making the superblock look like "just another entry" allows us to treat the superblock like "just another entry" and reuse a decent amount of logic that would otherwise only be used a format and mount time. In this case we can use append to write out the superblock like it was creating a new entry on the filesystem. --- lfs.c | 77 ++++++++++++++++++++++++------------------- lfs.h | 5 --- tests/test_corrupt.sh | 4 +-- tests/test_format.sh | 12 +------ 4 files changed, 46 insertions(+), 52 deletions(-) diff --git a/lfs.c b/lfs.c index 376e083a..4375ad98 100644 --- a/lfs.c +++ b/lfs.c @@ -2372,38 +2372,40 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs->root[0] = root.pair[0]; lfs->root[1] = root.pair[1]; + superdir.d.tail[0] = lfs->root[0]; + superdir.d.tail[1] = lfs->root[1]; - // write superblocks - lfs_superblock_t superblock = { - .d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK, - .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, - .d.nlen = sizeof(superblock.d.magic), - .d.version = LFS_DISK_VERSION, - .d.magic = {"littlefs"}, - .d.block_size = lfs->cfg->block_size, - .d.block_count = lfs->cfg->block_count, - .d.root = {lfs->root[0], lfs->root[1]}, - }; - superdir.d.tail[0] = root.pair[0]; - superdir.d.tail[1] = root.pair[1]; - superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; - - // write both pairs to be safe + // write one superblocks + lfs_superblock_t superblock; + superblock.d.version = LFS_DISK_VERSION, + superblock.d.root[0] = lfs->root[0]; + superblock.d.root[1] = lfs->root[1]; + superblock.d.block_size = lfs->cfg->block_size; + superblock.d.block_count = lfs->cfg->block_count; + + lfs_entry_t superentry; + superentry.d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK; + superentry.d.elen = sizeof(superblock.d); + superentry.d.alen = 0; + superentry.d.nlen = strlen("littlefs"); + superentry.off = sizeof(superdir.d); + superentry.size = 4 + superentry.d.elen + + superentry.d.alen + superentry.d.nlen; + + lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); - bool valid = false; - for (int i = 0; i < 2; i++) { - err = lfs_dir_commit(lfs, &superdir, &(struct lfs_region){ - sizeof(superdir.d), 0, - lfs_commit_mem, &superblock.d, sizeof(superblock.d)}); - if (err && err != LFS_ERR_CORRUPT) { - return err; - } - - valid = valid || !err; - } - - if (!valid) { - return LFS_ERR_CORRUPT; + err = lfs_dir_append(lfs, &superdir, &superentry, + &(struct lfs_region){ + 0, +4, + lfs_commit_mem, &superentry.d, 4, + &(struct lfs_region){ + 0, +sizeof(superblock.d), + lfs_commit_mem, &superblock.d, sizeof(superblock.d), + &(struct lfs_region){ + 0, +superentry.d.nlen, + lfs_commit_mem, "littlefs", superentry.d.nlen}}}); + if (err) { + return err; } // sanity check that fetch works @@ -2412,7 +2414,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - lfs_alloc_ack(lfs); return lfs_deinit(lfs); } @@ -2431,25 +2432,33 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; lfs_superblock_t superblock; + char magic[8]; err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err && err != LFS_ERR_CORRUPT) { return err; } if (!err) { - err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d), + err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } + err = lfs_bd_read(lfs, dir.pair[0], + sizeof(dir.d) + 4 + sizeof(superblock.d), + magic, sizeof(magic)); + if (err) { + return err; + } + lfs->root[0] = superblock.d.root[0]; lfs->root[1] = superblock.d.root[1]; } - if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", 0, 1); + if (err || memcmp(magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]); return LFS_ERR_CORRUPT; } diff --git a/lfs.h b/lfs.h index 75033b4c..6acf5e75 100644 --- a/lfs.h +++ b/lfs.h @@ -257,15 +257,10 @@ typedef struct lfs_dir { typedef struct lfs_superblock { struct lfs_disk_superblock { - uint8_t type; - uint8_t elen; - uint8_t alen; - uint8_t nlen; lfs_block_t root[2]; uint32_t block_size; uint32_t block_count; uint32_t version; - char magic[8]; } d; } lfs_superblock_t; diff --git a/tests/test_corrupt.sh b/tests/test_corrupt.sh index 6af984ff..1491dac3 100755 --- a/tests/test_corrupt.sh +++ b/tests/test_corrupt.sh @@ -73,7 +73,7 @@ lfs_mktree lfs_chktree echo "--- Block corruption ---" -for i in {0..33} +for i in {2..33} do rm -rf blocks mkdir blocks @@ -83,7 +83,7 @@ do done echo "--- Block persistance ---" -for i in {0..33} +for i in {2..33} do rm -rf blocks mkdir blocks diff --git a/tests/test_format.sh b/tests/test_format.sh index b9071015..cff0baca 100755 --- a/tests/test_format.sh +++ b/tests/test_format.sh @@ -30,20 +30,10 @@ echo "--- Invalid mount ---" tests/test.py << TEST lfs_format(&lfs, &cfg) => 0; TEST -rm blocks/0 blocks/1 +rm -f blocks/0 blocks/1 tests/test.py << TEST lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT; TEST -echo "--- Valid corrupt mount ---" -tests/test.py << TEST - lfs_format(&lfs, &cfg) => 0; -TEST -rm blocks/0 -tests/test.py << TEST - lfs_mount(&lfs, &cfg) => 0; - lfs_unmount(&lfs) => 0; -TEST - echo "--- Results ---" tests/stats.py From ad74825bcf9ba1f4ab4742f26515f6a2bb92d6d4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 23 Mar 2018 18:35:55 -0500 Subject: [PATCH 17/31] Added internal lfs_dir_get to consolidate logic for reading dir entries It's a relatively simple function but offers some code reuse as well as making the dir entry operations a bit more readable. --- lfs.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lfs.c b/lfs.c index 4375ad98..fdeebe08 100644 --- a/lfs.c +++ b/lfs.c @@ -677,6 +677,12 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } +// TODO zeros? +static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, + lfs_off_t off, void *buffer, lfs_size_t size) { + return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); +} + static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions) { // check if we fit, if top bit is set we do not and move on @@ -876,8 +882,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { dir->pos += sizeof(dir->d) + 4; } - int err = lfs_bd_read(lfs, dir->pair[0], dir->off, - &entry->d, sizeof(entry->d)); + int err = lfs_dir_get(lfs, dir, dir->off, &entry->d, sizeof(entry->d)); lfs_entry_fromle32(&entry->d); if (err) { return err; @@ -1167,7 +1172,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { info->size = entry.d.elen; } - int err = lfs_bd_read(lfs, dir->pair[0], + int err = lfs_dir_get(lfs, dir, entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { @@ -1502,7 +1507,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; - err = lfs_bd_read(lfs, cwd.pair[0], + err = lfs_dir_get(lfs, &cwd, entry.off + 4, file->cache.buffer, file->size); if (err) { @@ -1679,8 +1684,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; - err = lfs_bd_read(lfs, cwd.pair[0], entry.off, - &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, + entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; @@ -2047,7 +2052,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { strcpy(info->name, "/"); } else { - err = lfs_bd_read(lfs, cwd.pair[0], + err = lfs_dir_get(lfs, &cwd, entry.off + entry.size - entry.d.nlen, info->name, entry.d.nlen); if (err) { @@ -2439,14 +2444,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } if (!err) { - err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d)+4, - &superblock.d, sizeof(superblock.d)); + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } - err = lfs_bd_read(lfs, dir.pair[0], + err = lfs_dir_get(lfs, &dir, sizeof(dir.d) + 4 + sizeof(superblock.d), magic, sizeof(magic)); if (err) { @@ -2504,8 +2509,8 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { // iterate over contents lfs_entry_t entry; while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { - err = lfs_bd_read(lfs, dir.pair[0], dir.off, - &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, + dir.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; From 955545839bfdfde234bfaa366242fc943d7f5532 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 27 Mar 2018 17:57:07 -0500 Subject: [PATCH 18/31] Added internal lfs_dir_set, an umbrella to dir append/update/remove operations This move was surprisingly complex, but offers the ultimate opportunity for code reuse in terms of resizable entries. Instead of needing to provide separate functions for adding and removing entries, adding and removing entries can just be viewed as changing an entry's size to-and-from zero. Unfortunately, it's not _quite_ that simple, since append and remove hide some relatively complex operations for when directory blocks overflow or need to be cleaned up. However, with enough shoehorning, and a new committer type that allows specifying recursive commit lists (is this now a push-down automata?), it does seem to be possible to shove all of the entry update logic into a single function. Sidenote, I switched back to an enum-based DSL, since the addition of a recursive region opcode breaks the consistency of what needs to be passed to the DSL callback functions. It's much simpler to handle each opcode explicitly inside a recursive lfs_commit_region function. --- lfs.c | 555 ++++++++++++++++++++++++++-------------------------------- 1 file changed, 251 insertions(+), 304 deletions(-) diff --git a/lfs.c b/lfs.c index fdeebe08..8d3017c9 100644 --- a/lfs.c +++ b/lfs.c @@ -152,6 +152,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; LFS_ASSERT(block != 0xffffffff); + LFS_ASSERT(off + size <= lfs->cfg->block_size); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -483,83 +484,93 @@ static int lfs_dir_fetch(lfs_t *lfs, return 0; } -struct lfs_commit { - uint32_t crc; - lfs_block_t block; - lfs_off_t off; -}; - -static int lfs_commit(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size) { - lfs_crc(&c->crc, data, size); - int err = lfs_bd_prog(lfs, c->block, c->off, data, size); - c->off += size; - return err; -} - struct lfs_region { - lfs_off_t off; - lfs_ssize_t diff; + enum { + LFS_FROM_DROP, + LFS_FROM_MEM, + LFS_FROM_REGION, + } type; - int (*commit)(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size); - const void *data; - lfs_size_t size; - struct lfs_region *next; + lfs_off_t off; + const void *buffer; + lfs_ssize_t size; }; -static int lfs_commit_mem(lfs_t *lfs, struct lfs_commit *c, - const void *data, lfs_size_t size) { - return lfs_commit(lfs, c, data, size); -} - -struct lfs_commit_disk { +struct lfs_region_region { lfs_block_t block; lfs_off_t off; struct lfs_region *regions; + int count; }; -static int lfs_commit_disk(lfs_t *lfs, struct lfs_commit *c, - const void *p, lfs_size_t size) { - const struct lfs_commit_disk *d = p; - - struct lfs_region *r = d->regions; - lfs_off_t off = 0; - while (true) { - if (r && r->off == off) { - lfs_off_t orig = c->off; - int err = r->commit(lfs, c, r->data, r->size); - if (err) { - return err; +static int lfs_commit_region(lfs_t *lfs, + lfs_block_t oldblock, lfs_off_t oldoff, + lfs_block_t newblock, lfs_off_t newoff, + lfs_off_t regionoff, + const struct lfs_region *regions, int count, + lfs_size_t size, uint32_t *crc) { + int i = 0; + lfs_size_t end = newoff + size; + while (newoff < end) { + // commit from different types of regions + if (i < count && regions[i].off == oldoff - regionoff) { + switch (regions[i].type) { + case LFS_FROM_DROP: { + oldoff -= regions[i].size; + break; + } + case LFS_FROM_MEM: { + lfs_crc(crc, regions[i].buffer, regions[i].size); + int err = lfs_bd_prog(lfs, newblock, newoff, + regions[i].buffer, regions[i].size); + if (err) { + return err; + } + newoff += regions[i].size; + break; + } + case LFS_FROM_REGION: { + const struct lfs_region_region *disk = regions[i].buffer; + int err = lfs_commit_region(lfs, + disk->block, disk->off, + newblock, newoff, + disk->off, disk->regions, disk->count, + regions[i].size, crc); + if (err) { + return err; + } + newoff += regions[i].size; + break; + } } - off += (c->off - orig) - r->diff; - r = r->next; - } else if (off < size) { + i += 1; + } else { + // copy data from old block if not covered by region uint8_t data; - int err = lfs_bd_read(lfs, d->block, d->off + off, &data, 1); + int err = lfs_bd_read(lfs, oldblock, oldoff, &data, 1); if (err) { return err; } - err = lfs_commit(lfs, c, &data, 1); + lfs_crc(crc, &data, 1); + err = lfs_bd_prog(lfs, newblock, newoff, &data, 1); if (err) { return err; } - off += 1; - } else { - return 0; + oldoff += 1; + newoff += 1; } } + + return 0; } -// TODO handle overflowing reads (zero?) -static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, - struct lfs_region *regions) { +static int lfs_dif_commit(lfs_t *lfs, lfs_dir_t *dir, + const struct lfs_region *regions, int count) { // state for copying over const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; - lfs_size_t oldsize = (0x7fffffff & dir->d.size) - 4; bool relocated = false; // increment revision count @@ -567,8 +578,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); - for (struct lfs_region *r = regions; r; r = r->next) { - dir->d.size += r->diff; + for (int i = 0; i < count; i++) { + dir->d.size += regions[i].size; } while (true) { @@ -581,19 +592,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - struct lfs_commit c = { - .crc = 0xffffffff, - .block = dir->pair[0], - .off = 0, - }; - + // commit header + uint32_t crc = 0xffffffff; lfs_dir_tole32(&dir->d); - err = lfs_commit_disk(lfs, &c, &(struct lfs_commit_disk){ - oldpair[1], 0, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &dir->d, sizeof(dir->d), - regions}}, oldsize); + lfs_crc(&crc, &dir->d, sizeof(dir->d)); + err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -602,9 +605,25 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - c.crc = lfs_tole32(c.crc); - err = lfs_bd_prog(lfs, dir->pair[0], c.off, &c.crc, 4); - c.crc = lfs_fromle32(c.crc); + // commit region + err = lfs_commit_region(lfs, + dir->pair[1], sizeof(dir->d), + dir->pair[0], sizeof(dir->d), + 0, regions, count, + (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, + &crc); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // commit crc + crc = lfs_tole32(crc); + err = lfs_bd_prog(lfs, dir->pair[0], + (0x7fffffff & dir->d.size)-4, &crc, 4); + crc = lfs_fromle32(crc); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -628,7 +647,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (ncrc != c.crc) { + if (ncrc != crc) { goto relocate; } } @@ -683,61 +702,106 @@ static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); } -static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - // check if we fit, if top bit is set we do not and move on - while (true) { - if ((0x7fffffff & dir->d.size) + entry->size <= lfs->cfg->block_size) { - entry->off = dir->d.size - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; - } +static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, + struct lfs_region *regions, int count) { + lfs_ssize_t diff = 0; + for (int i = 0; i < count; i++) { + diff += regions[i].size; + } - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); - return err; - } + lfs_size_t oldsize = entry->size; + if (entry->off == 0) { + entry->off = (0x7fffffff & dir->d.size) - 4; + } - // we need to allocate a new dir block - if (!(0x80000000 & dir->d.size)) { - lfs_dir_t olddir = *dir; - int err = lfs_dir_alloc(lfs, dir); + if ((0x7fffffff & dir->d.size) + diff > lfs->cfg->block_size) { + lfs_dir_t olddir = *dir; + lfs_off_t oldoff = entry->off; + + if (oldsize) { + // mark as moving + uint8_t type; + int err = lfs_dir_get(lfs, &olddir, oldoff, &type, 1); + if (err) { + return err; + } + + type |= LFS_STRUCT_MOVED; + err = lfs_dif_commit(lfs, &olddir, (struct lfs_region[]){ + {LFS_FROM_MEM, oldoff, &type, 1}, + {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); if (err) { return err; } + } - dir->d.tail[0] = olddir.d.tail[0]; - dir->d.tail[1] = olddir.d.tail[1]; - entry->off = dir->d.size - 4; - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; + lfs_dir_t pdir = olddir; + + // find available block or create a new one + while ((0x7fffffff & dir->d.size) + oldsize + diff + > lfs->cfg->block_size) { + // we need to allocate a new dir block + if (!(0x80000000 & dir->d.size)) { + pdir = *dir; + int err = lfs_dir_alloc(lfs, dir); + if (err) { + return err; + } + + dir->d.tail[0] = pdir.d.tail[0]; + dir->d.tail[1] = pdir.d.tail[1]; + + break; } - lfs_entry_tole32(&entry->d); - err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); + int err = lfs_dir_fetch(lfs, dir, dir->d.tail); if (err) { return err; } + } + // writing out new entry + entry->off = dir->d.size - 4; + entry->size += diff; + int err = lfs_dif_commit(lfs, dir, (struct lfs_region[]){ + {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ + olddir.pair[0], oldoff, + regions, count}, entry->size}}, 1); + if (err) { + return err; + } + + // update pred dir, unless pred == old we can coalesce + if (!oldsize || lfs_paircmp(pdir.pair, olddir.pair) != 0) { + pdir.d.size |= 0x80000000; + pdir.d.tail[0] = dir->pair[0]; + pdir.d.tail[1] = dir->pair[1]; + + err = lfs_dif_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + } else if (oldsize) { olddir.d.size |= 0x80000000; olddir.d.tail[0] = dir->pair[0]; olddir.d.tail[1] = dir->pair[1]; - return lfs_dir_commit(lfs, &olddir, NULL); } - int err = lfs_dir_fetch(lfs, dir, dir->d.tail); - if (err) { - return err; + // remove old entry + if (oldsize) { + lfs_entry_t oldentry; + oldentry.off = oldoff; + err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -oldsize}}, 1); + if (err) { + return err; + } } + + goto shift; } -} -static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, - const lfs_entry_t *entry) { - // check if we should just drop the directory block - if ((dir->d.size & 0x7fffffff) == sizeof(dir->d)+4 + entry->size) { + if ((0x7fffffff & dir->d.size) + diff == sizeof(dir->d)+4) { lfs_dir_t pdir; int res = lfs_pred(lfs, dir->pair, &pdir); if (res < 0) { @@ -748,27 +812,34 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - return lfs_dir_commit(lfs, &pdir, NULL); + int err = lfs_dif_commit(lfs, &pdir, NULL, 0); + if (err) { + return err; + } + goto shift; } } - // shift out the entry - int err = lfs_dir_commit(lfs, dir, - &(struct lfs_region){ - entry->off, -entry->size, - lfs_commit_mem, NULL, 0}); + for (int i = 0; i < count; i++) { + regions[i].off += entry->off; + } + + int err = lfs_dif_commit(lfs, dir, regions, count); if (err) { return err; } + entry->size += diff; + +shift: // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off) { + if (f->poff == entry->off && entry->size == 0) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; } else if (f->poff > entry->off) { - f->poff -= entry->size; + f->poff += diff; } } } @@ -776,91 +847,10 @@ static int lfs_dir_remove(lfs_t *lfs, lfs_dir_t *dir, for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { if (lfs_paircmp(d->pair, dir->pair) == 0) { if (d->off > entry->off) { - d->off -= entry->size; - d->pos -= entry->size; - } - } - } - - return 0; -} - -static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, - lfs_entry_t *entry, struct lfs_region *regions) { - lfs_ssize_t diff = 0; - for (struct lfs_region *r = regions; r; r = r->next) { - diff += r->diff; - } - - // do we still fit? - if ((0x7fffffff & dir->d.size) + diff <= lfs->cfg->block_size) { - for (struct lfs_region *r = regions; r; r = r->next) { - r->off += entry->off; - } - - lfs_entry_tole32(&entry->d); - int err = lfs_dir_commit(lfs, dir, regions); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } - - // shift over any files/directories that are affected - for (lfs_file_t *f = lfs->files; f; f = f->next) { - if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff > entry->off) { - f->poff += diff; - } - } - } - - for (lfs_dir_t *d = lfs->dirs; d; d = d->next) { - if (lfs_paircmp(d->pair, dir->pair) == 0) { - if (d->off > entry->off) { - d->off += diff; - d->pos += diff; - } + d->off += diff; + d->pos += diff; } } - } else { - lfs_dir_t olddir = *dir; - lfs_entry_t oldentry = { - .off = entry->off, - .size = entry->size - diff, - .d.type = entry->d.type | LFS_STRUCT_MOVED, // TODO FIXME when changing type?? - }; - - // mark as moving - int err = lfs_dir_commit(lfs, &olddir, - &(struct lfs_region){ - oldentry.off, 0, - lfs_commit_mem, &oldentry.d.type, 1}); - if (err) { - return err; - } - - // append updated entry - lfs_entry_tole32(&entry->d); - err = lfs_dir_append(lfs, dir, entry, - &(struct lfs_region){ - 0, +entry->size, - lfs_commit_disk, &(struct lfs_commit_disk){ - olddir.pair[0], entry->off, regions}, oldentry.size}); - lfs_entry_fromle32(&entry->d); - if (err) { - return err; - } - - // remove old entry - err = lfs_dir_fetch(lfs, &olddir, olddir.pair); - if (err) { - return err; - } - - err = lfs_dir_remove(lfs, &olddir, &oldentry); - if (err) { - return err; - } } return 0; @@ -1039,7 +1029,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &dir, NULL); + err = lfs_dif_commit(lfs, &dir, NULL, 0); if (err) { return err; } @@ -1050,18 +1040,13 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.nlen = strlen(path); entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + entry.size = 0; cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; - - err = lfs_dir_append(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, +sizeof(entry.d), - lfs_commit_mem, &entry.d, sizeof(entry.d), - &(struct lfs_region){ - 0, +entry.d.nlen, - lfs_commit_mem, path, entry.d.nlen}}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); if (err) { return err; } @@ -1447,15 +1432,11 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.d.elen = 0; entry.d.alen = 0; entry.d.nlen = strlen(path); - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + entry.size = 0; - err = lfs_dir_append(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, +4, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 0, +entry.d.nlen, - lfs_commit_mem, path, entry.d.nlen}}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); if (err) { return err; } @@ -1684,43 +1665,37 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; - err = lfs_dir_get(lfs, &cwd, - entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; } + LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); + lfs_size_t oldlen = entry.d.elen; + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { - lfs_ssize_t diff = sizeof(entry.d)-4 - entry.d.elen; entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; entry.d.elen = sizeof(entry.d)-4; entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - // TODO combine down? - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, diff, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 2); if (err) { return err; } } else { - lfs_ssize_t diff = file->size - entry.d.elen; entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = file->size; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, 4, - &(struct lfs_region){ - 4, diff, - lfs_commit_mem, file->cache.buffer, file->size}}); + + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, file->cache.buffer, file->size}, + {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 3); if (err) { return err; } @@ -2098,7 +2073,8 @@ int lfs_remove(lfs_t *lfs, const char *path) { } // remove the entry - err = lfs_dir_remove(lfs, &cwd, &entry); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); if (err) { return err; } @@ -2114,7 +2090,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &cwd, NULL); + err = lfs_dif_commit(lfs, &cwd, NULL, 0); if (err) { return err; } @@ -2181,10 +2157,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &oldcwd, &oldentry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &oldentry.d.type, 1}); + err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &oldentry.d.type, 1}, + {LFS_FROM_DROP, 0, NULL, -1}}, 2); oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; @@ -2200,42 +2175,22 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = strlen(newpath); - newentry.size = 4 + newentry.d.elen + newentry.d.alen + newentry.d.nlen; - - if (prevexists) { - err = lfs_dir_update(lfs, &newcwd, &newentry, - &(struct lfs_region){ - 0, newentry.size - preventry.size, - lfs_commit_disk, &(struct lfs_commit_disk){ - oldcwd.pair[0], oldentry.off, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, 4, - &(struct lfs_region){ - newentry.size - newentry.d.nlen, - +newentry.d.nlen-oldentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}}, - oldentry.size}); - if (err) { - return err; - } - } else { - err = lfs_dir_append(lfs, &newcwd, &newentry, - &(struct lfs_region){ - 0, +newentry.size, - lfs_commit_disk, &(struct lfs_commit_disk){ - oldcwd.pair[0], oldentry.off, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &newentry.d, 4, - &(struct lfs_region){ - newentry.size - newentry.d.nlen, - +newentry.d.nlen-oldentry.d.nlen, - lfs_commit_mem, newpath, newentry.d.nlen}}}, - oldentry.size}); - if (err) { - return err; - } + if (!prevexists) { + newentry.size = 0; + } + + lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; + err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ + {LFS_FROM_REGION, 0, &(struct lfs_region_region){ + oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &newentry.d, 4}, + {LFS_FROM_DROP, 0, NULL, -4}, + {LFS_FROM_MEM, newsize - newentry.d.nlen, + newpath, newentry.d.nlen}}, 3}, + newsize}, + {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); + if (err) { + return err; } // update pair if newcwd == oldcwd @@ -2244,7 +2199,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } // remove old entry - err = lfs_dir_remove(lfs, &oldcwd, &oldentry); + err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -oldentry.size}}, 1); if (err) { return err; } @@ -2260,7 +2216,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dir_commit(lfs, &newcwd, NULL); + err = lfs_dif_commit(lfs, &newcwd, NULL, 0); if (err) { return err; } @@ -2370,7 +2326,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_commit(lfs, &root, NULL); + err = lfs_dif_commit(lfs, &root, NULL, 0); if (err) { return err; } @@ -2394,21 +2350,14 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { superentry.d.alen = 0; superentry.d.nlen = strlen("littlefs"); superentry.off = sizeof(superdir.d); - superentry.size = 4 + superentry.d.elen + - superentry.d.alen + superentry.d.nlen; + superentry.size = 0; lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); - err = lfs_dir_append(lfs, &superdir, &superentry, - &(struct lfs_region){ - 0, +4, - lfs_commit_mem, &superentry.d, 4, - &(struct lfs_region){ - 0, +sizeof(superblock.d), - lfs_commit_mem, &superblock.d, sizeof(superblock.d), - &(struct lfs_region){ - 0, +superentry.d.nlen, - lfs_commit_mem, "littlefs", superentry.d.nlen}}}); + err = lfs_dir_set(lfs, &superdir, &superentry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &superentry.d, 4}, + {LFS_FROM_MEM, 0, &superblock.d, sizeof(superblock.d)}, + {LFS_FROM_MEM, 0, "littlefs", superentry.d.nlen}}, 3); if (err) { return err; } @@ -2452,8 +2401,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } err = lfs_dir_get(lfs, &dir, - sizeof(dir.d) + 4 + sizeof(superblock.d), - magic, sizeof(magic)); + sizeof(dir.d)+4 + sizeof(superblock.d), magic, sizeof(magic)); if (err) { return err; } @@ -2671,11 +2619,9 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; - - int err = lfs_dir_update(lfs, &parent, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, (lfs_ssize_t)-sizeof(entry.d)}}, 2); if (err) { return err; } @@ -2702,7 +2648,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dir_commit(lfs, &parent, NULL); + return lfs_dif_commit(lfs, &parent, NULL, 0); } // couldn't find dir, must be new @@ -2744,7 +2690,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dir_commit(lfs, &pdir, NULL); + err = lfs_dif_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2760,7 +2706,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dir_commit(lfs, &pdir, NULL); + err = lfs_dif_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2791,7 +2737,8 @@ int lfs_deorphan(lfs_t *lfs) { if (moved) { LFS_DEBUG("Found move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); - err = lfs_dir_remove(lfs, &cwd, &entry); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); if (err) { return err; } @@ -2799,10 +2746,10 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found partial move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_DROP, 0, NULL, + (lfs_ssize_t)-sizeof(entry.d)}}, 2); if (err) { return err; } From 6362afa8d06da19f9ce9b1b5b62100ec8a16031d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 1 Apr 2018 15:36:29 -0500 Subject: [PATCH 19/31] Added disk-backed limits on the name/attrs/inline sizes Being a portable, microcontroller-scale embedded filesystem, littlefs is presented with a relatively unique challenge. The amount of RAM available is on completely different scales from machine to machine, and what is normally a reasonable RAM assumption may break completely on an embedded system. A great example of this is file names. On almost every PC these days, the limit for a file name is 255 bytes. It's a very convenient limit for a number of reasons. However, on microcontrollers, allocating 255 bytes of RAM to do a file search can be unreasonable. The simplest solution (and one that has existing in littlefs for a while), is to let this limit be redefined to a smaller value on devices that need to save RAM. However, this presents an interesting portability issue. If these devices are plugged into a PC with relatively infinite RAM, nothing stops the PC from writing files with full 255-byte file names, which can't be read on the small device. One solution here is to store this limit on the superblock during format time. When mounting a disk, the filesystem implementation is responsible for checking this limit in the superblock. If it's larger than what can be read, raise an error. If it's smaller, respect the limit on the superblock and raise an error if the user attempts to exceed it. In this commit, this strategy is adopted for file names, inline files, and the size of all attributes, since these could impact the memory consumption of the filesystem. (Recording the attribute's limit is iffy, but is the only other arbitrary limit and could be used for disabling support of custom attributes). Note! This changes makes it very important to configure littlefs correctly at format time. If littlefs is formatted on a PC without changing the limits appropriately, it will be rejected by a smaller device. --- lfs.c | 146 +++++++++++++++++++++++++++++++++++++++++++++------------- lfs.h | 67 ++++++++++++++++++--------- 2 files changed, 159 insertions(+), 54 deletions(-) diff --git a/lfs.c b/lfs.c index 8d3017c9..ad3ad041 100644 --- a/lfs.c +++ b/lfs.c @@ -365,6 +365,9 @@ static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { d->block_size = lfs_fromle32(d->block_size); d->block_count = lfs_fromle32(d->block_count); d->version = lfs_fromle32(d->version); + d->inline_size = lfs_fromle32(d->inline_size); + d->attrs_size = lfs_fromle32(d->attrs_size); + d->name_size = lfs_fromle32(d->name_size); } static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { @@ -373,6 +376,9 @@ static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { d->block_size = lfs_tole32(d->block_size); d->block_count = lfs_tole32(d->block_count); d->version = lfs_tole32(d->version); + d->inline_size = lfs_tole32(d->inline_size); + d->attrs_size = lfs_tole32(d->attrs_size); + d->name_size = lfs_tole32(d->name_size); } @@ -1018,6 +1024,12 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { return err ? err : LFS_ERR_EXIST; } + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // build up new directory lfs_alloc_ack(lfs); @@ -1037,7 +1049,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { entry.d.type = LFS_STRUCT_DIR | LFS_TYPE_DIR; entry.d.elen = sizeof(entry.d) - 4; entry.d.alen = 0; - entry.d.nlen = strlen(path); + entry.d.nlen = nlen; entry.d.u.dir[0] = dir.pair[0]; entry.d.u.dir[1] = dir.pair[1]; entry.size = 0; @@ -1046,7 +1058,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[1] = dir.pair[1]; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); + {LFS_FROM_MEM, 0, path, nlen}}, 2); if (err) { return err; } @@ -1427,16 +1439,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_NOENT; } + // check that name fits + lfs_size_t nlen = strlen(path); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // create entry to remember name entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; entry.d.elen = 0; entry.d.alen = 0; - entry.d.nlen = strlen(path); + entry.d.nlen = nlen; entry.size = 0; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, path, entry.d.nlen}}, 2); + {LFS_FROM_MEM, 0, path, nlen}}, 2); if (err) { return err; } @@ -1571,10 +1589,6 @@ relocate:; static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { if (file->flags & LFS_F_READING) { - if (!(file->flags & LFS_F_INLINE)) { - // just drop read cache - file->cache.block = 0xffffffff; - } file->flags &= ~LFS_F_READING; } @@ -1807,9 +1821,8 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO need to move out if no longer fits in block also // TODO store INLINE_MAX in superblock? // TODO what if inline files is > block size (ie 128) - if ((file->flags & LFS_F_INLINE) && ( - (file->pos + nsize >= LFS_INLINE_MAX) || - (file->pos + nsize >= lfs->cfg->read_size))) { + if ((file->flags & LFS_F_INLINE) && + file->pos + nsize >= lfs->inline_size) { file->block = 0xfffffffe; file->off = file->pos; @@ -2137,6 +2150,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { bool prevexists = (err != LFS_ERR_NOENT); bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); + // check that name fits + lfs_size_t nlen = strlen(newpath); + if (nlen > lfs->name_size) { + return LFS_ERR_NAMETOOLONG; + } + // must have same type if (prevexists && preventry.d.type != oldentry.d.type) { return LFS_ERR_ISDIR; @@ -2174,7 +2193,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { lfs_entry_t newentry = preventry; newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; - newentry.d.nlen = strlen(newpath); + newentry.d.nlen = nlen; if (!prevexists) { newentry.size = 0; } @@ -2185,8 +2204,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &newentry.d, 4}, {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_MEM, newsize - newentry.d.nlen, - newpath, newentry.d.nlen}}, 3}, + {LFS_FROM_MEM, newsize - nlen, newpath, nlen}}, 3}, newsize}, {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); if (err) { @@ -2272,6 +2290,26 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) <= lfs->cfg->block_size); + // check that the size limits are sane + LFS_ASSERT(lfs->cfg->inline_size <= LFS_INLINE_MAX); + LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->read_size); + lfs->inline_size = lfs->cfg->inline_size; + if (!lfs->inline_size) { + lfs->inline_size = lfs_min(LFS_INLINE_MAX, lfs->cfg->read_size); + } + + LFS_ASSERT(lfs->cfg->attrs_size <= LFS_ATTRS_MAX); + lfs->attrs_size = lfs->cfg->attrs_size; + if (!lfs->attrs_size) { + lfs->attrs_size = LFS_ATTRS_MAX; + } + + LFS_ASSERT(lfs->cfg->name_size <= LFS_NAME_MAX); + lfs->name_size = lfs->cfg->name_size; + if (!lfs->name_size) { + lfs->name_size = LFS_NAME_MAX; + } + // setup default state lfs->root[0] = 0xffffffff; lfs->root[1] = 0xffffffff; @@ -2336,13 +2374,16 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { superdir.d.tail[0] = lfs->root[0]; superdir.d.tail[1] = lfs->root[1]; - // write one superblocks + // write one superblock lfs_superblock_t superblock; superblock.d.version = LFS_DISK_VERSION, superblock.d.root[0] = lfs->root[0]; superblock.d.root[1] = lfs->root[1]; superblock.d.block_size = lfs->cfg->block_size; superblock.d.block_count = lfs->cfg->block_count; + superblock.d.inline_size = lfs->inline_size; + superblock.d.attrs_size = lfs->attrs_size; + superblock.d.name_size = lfs->name_size; lfs_entry_t superentry; superentry.d.type = LFS_STRUCT_DIR | LFS_TYPE_SUPERBLOCK; @@ -2385,33 +2426,41 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; + lfs_entry_t entry; lfs_superblock_t superblock; char magic[8]; + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); - if (err && err != LFS_ERR_CORRUPT) { + if (err) { + if (err == LFS_ERR_CORRUPT) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); + } return err; } - if (!err) { - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4, &superblock.d, sizeof(superblock.d)); - lfs_superblock_fromle32(&superblock.d); - if (err) { - return err; - } + err = lfs_dir_get(lfs, &dir, sizeof(dir.d), &entry.d, sizeof(entry.d)); + if (err) { + return err; + } - err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4 + sizeof(superblock.d), magic, sizeof(magic)); - if (err) { - return err; - } + memset(&superblock.d, 0, sizeof(superblock.d)); + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4, &superblock.d, + lfs_min(sizeof(superblock.d), entry.d.elen)); + lfs_superblock_fromle32(&superblock.d); + if (err) { + return err; + } - lfs->root[0] = superblock.d.root[0]; - lfs->root[1] = superblock.d.root[1]; + err = lfs_dir_get(lfs, &dir, + sizeof(dir.d)+4+entry.d.elen+entry.d.alen, magic, + lfs_min(sizeof(magic), entry.d.nlen)); + if (err) { + return err; } - if (err || memcmp(magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]); + if (memcmp(magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); return LFS_ERR_CORRUPT; } @@ -2423,6 +2472,39 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return LFS_ERR_INVAL; } + if (superblock.d.inline_size) { + if (superblock.d.inline_size > lfs->inline_size) { + LFS_ERROR("Unsupported inline size (%d > %d)", + superblock.d.inline_size, lfs->inline_size); + return LFS_ERR_INVAL; + } + + lfs->inline_size = superblock.d.inline_size; + } + + if (superblock.d.attrs_size) { + if (superblock.d.attrs_size > lfs->attrs_size) { + LFS_ERROR("Unsupported attrs size (%d > %d)", + superblock.d.attrs_size, lfs->attrs_size); + return LFS_ERR_INVAL; + } + + lfs->attrs_size = superblock.d.attrs_size; + } + + if (superblock.d.name_size) { + if (superblock.d.name_size > lfs->name_size) { + LFS_ERROR("Unsupported name size (%d > %d)", + superblock.d.name_size, lfs->name_size); + return LFS_ERR_INVAL; + } + + lfs->name_size = superblock.d.name_size; + } + + lfs->root[0] = superblock.d.root[0]; + lfs->root[1] = superblock.d.root[1]; + return 0; } diff --git a/lfs.h b/lfs.h index 6acf5e75..9a7b8c37 100644 --- a/lfs.h +++ b/lfs.h @@ -50,30 +50,37 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; +// Maximum inline file size in bytes +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 255 +#endif + +// Maximum size of all attributes per file in bytes +#ifndef LFS_ATTRS_MAX +#define LFS_ATTRS_MAX 255 +#endif + // Max name size in bytes #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 255 #endif -#ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 255 -#endif - // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { - LFS_ERR_OK = 0, // No error - LFS_ERR_IO = -5, // Error during device operation - LFS_ERR_CORRUPT = -52, // Corrupted - LFS_ERR_NOENT = -2, // No directory entry - LFS_ERR_EXIST = -17, // Entry already exists - LFS_ERR_NOTDIR = -20, // Entry is not a dir - LFS_ERR_ISDIR = -21, // Entry is a dir - LFS_ERR_NOTEMPTY = -39, // Dir is not empty - LFS_ERR_BADF = -9, // Bad file number - LFS_ERR_INVAL = -22, // Invalid parameter - LFS_ERR_NOSPC = -28, // No space left on device - LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_OK = 0, // No error + LFS_ERR_IO = -5, // Error during device operation + LFS_ERR_CORRUPT = -52, // Corrupted + LFS_ERR_NOENT = -2, // No directory entry + LFS_ERR_EXIST = -17, // Entry already exists + LFS_ERR_NOTDIR = -20, // Entry is not a dir + LFS_ERR_ISDIR = -21, // Entry is a dir + LFS_ERR_NOTEMPTY = -39, // Dir is not empty + LFS_ERR_BADF = -9, // Bad file number + LFS_ERR_INVAL = -22, // Invalid parameter + LFS_ERR_NOSPC = -28, // No space left on device + LFS_ERR_NOMEM = -12, // No more memory available + LFS_ERR_NAMETOOLONG = -36, // File name too long }; // File types @@ -102,10 +109,10 @@ enum lfs_open_flags { LFS_O_APPEND = 0x0800, // Move to end of file on every write // internally used flags - LFS_F_DIRTY = 0x10000, // File does not match storage - LFS_F_WRITING = 0x20000, // File has been written since last flush - LFS_F_READING = 0x40000, // File has been read since last flush - LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_DIRTY = 0x010000, // File does not match storage + LFS_F_WRITING = 0x020000, // File has been written since last flush + LFS_F_READING = 0x040000, // File has been read since last flush + LFS_F_ERRED = 0x080000, // An error occured during write LFS_F_INLINE = 0x100000, // Currently inlined in directory entry }; @@ -183,6 +190,13 @@ struct lfs_config { // Optional, statically allocated buffer for files. Must be program sized. // If enabled, only one file may be opened at a time. void *file_buffer; + + // Optional, + lfs_size_t inline_size; + // Optional, + lfs_size_t attrs_size; + // Optional, + lfs_size_t name_size; }; @@ -258,9 +272,14 @@ typedef struct lfs_dir { typedef struct lfs_superblock { struct lfs_disk_superblock { lfs_block_t root[2]; - uint32_t block_size; - uint32_t block_count; + + lfs_size_t block_size; + lfs_size_t block_count; uint32_t version; + + lfs_size_t inline_size; + lfs_size_t attrs_size; + lfs_size_t name_size; } d; } lfs_superblock_t; @@ -285,6 +304,10 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; + + lfs_size_t inline_size; + lfs_size_t attrs_size; + lfs_size_t name_size; } lfs_t; From 67742761245597833bdad497083b3319ce5d71d4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 3 Apr 2018 08:28:09 -0500 Subject: [PATCH 20/31] Expanded inline files up to a limit of 1023 bytes One of the big benefits of inline files is that small files no longer need to take up a full block. This opens up an opportunity to provide much better support for storage devices with only a handful of very large blocks. Such as the internal flash found on most microcontrollers. After investigating some use cases for a filesystem on internal flash, it has become apparent that the 255-byte limit is going to be too restrictive to be useful in many cases. Most uses I found needed files ~4-64 bytes in size, but it wasn't uncommon to find files ~512 bytes in length. To try to remedy this, I've pushed the 255 byte limit up to 1023 bytes, by stealing some bits from the previously-unused attributes's size. Unfortunately this limits attributes to 63 bytes in total and has a minor code cost, but I'm not sure even 1023 bytes will be sufficient for a lot of cases. The littlefs will probably never be as efficient with internal flash as other filesystems such as SPIFFS, it just wasn't designed for this sort of limited geometry. However, this feature has been heavily requested, even with limitations, because of the opportunity for code reuse on microcontrollers with both internal and external flash. --- lfs.c | 103 ++++++++++++++++++++++++++++++++++++++-------------------- lfs.h | 7 ++-- 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/lfs.c b/lfs.c index ad3ad041..d5342c05 100644 --- a/lfs.c +++ b/lfs.c @@ -381,6 +381,26 @@ static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { d->name_size = lfs_tole32(d->name_size); } +/// Other struct functions /// +static inline lfs_size_t lfs_entry_elen(const lfs_entry_t *entry) { + return (lfs_size_t)(entry->d.elen) | + ((lfs_size_t)(entry->d.alen & 0xc0) << 2); +} + +static inline lfs_size_t lfs_entry_alen(const lfs_entry_t *entry) { + return entry->d.alen & 0x3f; +} + +static inline lfs_size_t lfs_entry_nlen(const lfs_entry_t *entry) { + return entry->d.nlen; +} + +static inline lfs_size_t lfs_entry_size(const lfs_entry_t *entry) { + return 4 + lfs_entry_elen(entry) + + lfs_entry_alen(entry) + + lfs_entry_nlen(entry); +} + /// Metadata pair and directory operations /// static inline void lfs_pairswap(lfs_block_t pair[2]) { @@ -573,7 +593,7 @@ static int lfs_commit_region(lfs_t *lfs, return 0; } -static int lfs_dif_commit(lfs_t *lfs, lfs_dir_t *dir, +static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, const struct lfs_region *regions, int count) { // state for copying over const lfs_block_t oldpair[2] = {dir->pair[1], dir->pair[0]}; @@ -733,7 +753,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, } type |= LFS_STRUCT_MOVED; - err = lfs_dif_commit(lfs, &olddir, (struct lfs_region[]){ + err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ {LFS_FROM_MEM, oldoff, &type, 1}, {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); if (err) { @@ -769,7 +789,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, // writing out new entry entry->off = dir->d.size - 4; entry->size += diff; - int err = lfs_dif_commit(lfs, dir, (struct lfs_region[]){ + int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ olddir.pair[0], oldoff, regions, count}, entry->size}}, 1); @@ -783,7 +803,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, pdir.d.tail[0] = dir->pair[0]; pdir.d.tail[1] = dir->pair[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -818,7 +838,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, pdir.d.size &= dir->d.size | 0x7fffffff; pdir.d.tail[0] = dir->d.tail[0]; pdir.d.tail[1] = dir->d.tail[1]; - int err = lfs_dif_commit(lfs, &pdir, NULL, 0); + int err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -830,7 +850,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, regions[i].off += entry->off; } - int err = lfs_dif_commit(lfs, dir, regions, count); + int err = lfs_dir_commit(lfs, dir, regions, count); if (err) { return err; } @@ -885,7 +905,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { } entry->off = dir->off; - entry->size = 4 + entry->d.elen + entry->d.alen + entry->d.nlen; + entry->size = lfs_entry_size(entry); dir->off += entry->size; dir->pos += entry->size; return 0; @@ -1041,7 +1061,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { dir.d.tail[0] = cwd.d.tail[0]; dir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dif_commit(lfs, &dir, NULL, 0); + err = lfs_dir_commit(lfs, &dir, NULL, 0); if (err) { return err; } @@ -1166,7 +1186,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = entry.d.elen; + info->size = lfs_entry_elen(&entry); } int err = lfs_dir_get(lfs, dir, @@ -1480,29 +1500,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } - // TODO combine these below? // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; file->poff = entry.off; - file->head = entry.d.u.file.head; - file->size = entry.d.u.file.size; file->flags = flags; file->pos = 0; - if (flags & LFS_O_TRUNC) { - if (file->size != 0) { - file->flags |= LFS_F_DIRTY; - } - - entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = 0; - } + // calculate max inline size based on the size of the entry + file->inline_size = lfs_min(lfs->inline_size, + lfs->cfg->block_size - (sizeof(cwd.d)+4) - + (lfs_entry_size(&entry) - lfs_entry_elen(&entry))); - // load inline files if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + // load inline files file->head = 0xfffffffe; - file->size = entry.d.elen; + file->size = lfs_entry_elen(&entry); file->flags |= LFS_F_INLINE; file->cache.block = file->head; file->cache.off = 0; @@ -1513,6 +1526,23 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, lfs_free(file->cache.buffer); return err; } + } else { + // use ctz list from entry + file->head = entry.d.u.file.head; + file->size = entry.d.u.file.size; + } + + // truncate if requested + if (flags & LFS_O_TRUNC) { + if (file->size != 0) { + file->flags |= LFS_F_DIRTY; + } + + file->head = 0xfffffffe; + file->size = 0; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; } // add to list of files @@ -1686,8 +1716,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - lfs_size_t oldlen = entry.d.elen; - entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + lfs_size_t oldlen = lfs_entry_elen(&entry); + entry.size = lfs_entry_size(&entry); // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { @@ -1704,7 +1734,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } } else { entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size; + entry.d.elen = file->size & 0xff; + entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, &entry.d, 4}, @@ -1822,7 +1853,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, // TODO store INLINE_MAX in superblock? // TODO what if inline files is > block size (ie 128) if ((file->flags & LFS_F_INLINE) && - file->pos + nsize >= lfs->inline_size) { + file->pos + nsize >= file->inline_size) { file->block = 0xfffffffe; file->off = file->pos; @@ -2034,7 +2065,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = entry.d.elen; + info->size = lfs_entry_elen(&entry); } if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { @@ -2103,7 +2134,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dif_commit(lfs, &cwd, NULL, 0); + err = lfs_dir_commit(lfs, &cwd, NULL, 0); if (err) { return err; } @@ -2234,7 +2265,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; - err = lfs_dif_commit(lfs, &newcwd, NULL, 0); + err = lfs_dir_commit(lfs, &newcwd, NULL, 0); if (err) { return err; } @@ -2364,7 +2395,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dif_commit(lfs, &root, NULL, 0); + err = lfs_dir_commit(lfs, &root, NULL, 0); if (err) { return err; } @@ -2446,14 +2477,14 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { memset(&superblock.d, 0, sizeof(superblock.d)); err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+4, &superblock.d, - lfs_min(sizeof(superblock.d), entry.d.elen)); + lfs_min(sizeof(superblock.d), lfs_entry_elen(&entry))); lfs_superblock_fromle32(&superblock.d); if (err) { return err; } err = lfs_dir_get(lfs, &dir, - sizeof(dir.d)+4+entry.d.elen+entry.d.alen, magic, + sizeof(dir.d)+lfs_entry_size(&entry)-entry.d.nlen, magic, lfs_min(sizeof(magic), entry.d.nlen)); if (err) { return err; @@ -2546,7 +2577,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { return err; } - dir.off += 4 + entry.d.elen + entry.d.alen + entry.d.nlen; + dir.off += lfs_entry_size(&entry); if ((0x70 & entry.d.type) == LFS_STRUCT_CTZ) { err = lfs_ctz_traverse(lfs, &lfs->rcache, NULL, entry.d.u.file.head, entry.d.u.file.size, cb, data); @@ -2730,7 +2761,7 @@ static int lfs_relocate(lfs_t *lfs, parent.d.tail[0] = newpair[0]; parent.d.tail[1] = newpair[1]; - return lfs_dif_commit(lfs, &parent, NULL, 0); + return lfs_dir_commit(lfs, &parent, NULL, 0); } // couldn't find dir, must be new @@ -2772,7 +2803,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = cwd.d.tail[0]; pdir.d.tail[1] = cwd.d.tail[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } @@ -2788,7 +2819,7 @@ int lfs_deorphan(lfs_t *lfs) { pdir.d.tail[0] = entry.d.u.dir[0]; pdir.d.tail[1] = entry.d.u.dir[1]; - err = lfs_dif_commit(lfs, &pdir, NULL, 0); + err = lfs_dir_commit(lfs, &pdir, NULL, 0); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 9a7b8c37..ca40e0f8 100644 --- a/lfs.h +++ b/lfs.h @@ -52,17 +52,17 @@ typedef uint32_t lfs_block_t; // Maximum inline file size in bytes #ifndef LFS_INLINE_MAX -#define LFS_INLINE_MAX 255 +#define LFS_INLINE_MAX 0x3ff #endif // Maximum size of all attributes per file in bytes #ifndef LFS_ATTRS_MAX -#define LFS_ATTRS_MAX 255 +#define LFS_ATTRS_MAX 0x3f #endif // Max name size in bytes #ifndef LFS_NAME_MAX -#define LFS_NAME_MAX 255 +#define LFS_NAME_MAX 0xff #endif // Possible error codes, these are negative to allow @@ -248,6 +248,7 @@ typedef struct lfs_file { lfs_size_t size; uint32_t flags; + lfs_size_t inline_size; lfs_off_t pos; lfs_block_t block; lfs_off_t off; From 65ea6b3d0f6bda68a3139fae1b1c2eb0b7882cdb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 3 Apr 2018 08:29:28 -0500 Subject: [PATCH 21/31] Bumped versions, cleaned up some TODOs and missing comments --- lfs.c | 14 +------------- lfs.h | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/lfs.c b/lfs.c index d5342c05..d8693767 100644 --- a/lfs.c +++ b/lfs.c @@ -722,7 +722,6 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, return 0; } -// TODO zeros? static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, lfs_off_t off, void *buffer, lfs_size_t size) { return lfs_bd_read(lfs, dir->pair[0], off, buffer, size); @@ -1180,8 +1179,6 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } - // TODO common info constructor? - // TODO also used in lfs_stat info->type = 0xf & entry.d.type; if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; @@ -1707,7 +1704,6 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - // TODO entry read function? lfs_entry_t entry = {.off = file->poff}; err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); @@ -1778,7 +1774,6 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, nsize = size; while (nsize > 0) { - // TODO can this be collapsed? // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { @@ -1848,12 +1843,9 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } } - // TODO combine with block allocation? - // TODO need to move out if no longer fits in block also - // TODO store INLINE_MAX in superblock? - // TODO what if inline files is > block size (ie 128) if ((file->flags & LFS_F_INLINE) && file->pos + nsize >= file->inline_size) { + // inline file doesn't fit anymore file->block = 0xfffffffe; file->off = file->pos; @@ -1869,9 +1861,6 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } while (nsize > 0) { - // TODO can this be collapsed? - // TODO can we reduce this now that block 0 is never allocated? - // TODO actually, how does this behave if inline max == 0? // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { @@ -1969,7 +1958,6 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, return file->pos; } -// TODO handle inlining? int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { if ((file->flags & 3) == LFS_O_RDONLY) { return LFS_ERR_BADF; diff --git a/lfs.h b/lfs.h index ca40e0f8..55148231 100644 --- a/lfs.h +++ b/lfs.h @@ -27,14 +27,14 @@ // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00010003 +#define LFS_VERSION 0x00010004 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) // Version of On-disk data structures // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_DISK_VERSION 0x00010001 +#define LFS_DISK_VERSION 0x00010002 #define LFS_DISK_VERSION_MAJOR (0xffff & (LFS_DISK_VERSION >> 16)) #define LFS_DISK_VERSION_MINOR (0xffff & (LFS_DISK_VERSION >> 0)) @@ -50,17 +50,25 @@ typedef int32_t lfs_soff_t; typedef uint32_t lfs_block_t; -// Maximum inline file size in bytes +// Maximum inline file size in bytes. Large inline files require a larger +// read and prog cache, but if a file can be inline it does not need its own +// data block. LFS_ATTRS_MAX + LFS_INLINE_MAX must be <= 0xffff. Stored in +// superblock and must be respected by other littlefs drivers. #ifndef LFS_INLINE_MAX #define LFS_INLINE_MAX 0x3ff #endif -// Maximum size of all attributes per file in bytes +// Maximum size of all attributes per file in bytes, may be redefined but a +// a smaller LFS_ATTRS_MAX has no benefit. LFS_ATTRS_MAX + LFS_INLINE_MAX +// must be <= 0xffff. Stored in superblock and must be respected by other +// littlefs drivers. #ifndef LFS_ATTRS_MAX #define LFS_ATTRS_MAX 0x3f #endif -// Max name size in bytes +// Max name size in bytes, may be redefined to reduce the size of the +// info struct. Stored in superblock and must be respected by other +// littlefs drivers. #ifndef LFS_NAME_MAX #define LFS_NAME_MAX 0xff #endif @@ -191,11 +199,23 @@ struct lfs_config { // If enabled, only one file may be opened at a time. void *file_buffer; - // Optional, + // Optional upper limit on inlined files in bytes. Large inline files + // require a larger read and prog cache, but if a file can be inlined it + // does not need its own data block. Must be smaller than the read size + // and prog size. Defaults to min(LFS_INLINE_MAX, read_size) when zero. + // Stored in superblock and must be respected by other littlefs drivers. lfs_size_t inline_size; - // Optional, + + // Optional upper limit on attributes per file in bytes. No downside for + // larger attributes size but must be less than LFS_ATTRS_MAX. Defaults to + // LFS_ATTRS_MAX when zero.Stored in superblock and must be respected by + // other littlefs drivers. lfs_size_t attrs_size; - // Optional, + + // Optional upper limit on length of file names in bytes. No downside for + // larger names except the size of the info struct which is controlled by + // the LFS_NAME_MAX define. Defaults to LFS_NAME_MAX when zero. Stored in + // superblock and must be respected by other littlefs drivers. lfs_size_t name_size; }; From 6ffc8d3480b01ee36001826dbe0489ac7badb761 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 5 Apr 2018 19:03:58 -0500 Subject: [PATCH 22/31] Added simple custom attributes A much requested feature (mostly because of littlefs's notable lack of timestamps), this commits adds support for user-specified custom attributes. Planned (though underestimated) since v1, custom attributes provide a route for OSs and applications to provide their own metadata in littlefs, without limiting portability. However, unlike custom attributes that can be found on much more powerful PC filesystems, these custom attributes are very limited, intended for only a handful of bytes for very important metadata. Each attribute has only a single byte to identify the attribute, and the size of all attributes attached to a file is limited to 64 bytes. Custom attributes can be accessed through the lfs_getattr, lfs_setattr, and lfs_removeattr functions. --- lfs.c | 224 +++++++++++++++++++++++++++++++++++++++++++++++++--------- lfs.h | 32 +++++++++ 2 files changed, 224 insertions(+), 32 deletions(-) diff --git a/lfs.c b/lfs.c index d8693767..fa63e8ee 100644 --- a/lfs.c +++ b/lfs.c @@ -913,7 +913,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const char **path) { const char *pathname = *path; - size_t pathlen; + lfs_size_t pathlen; while (true) { nextname: @@ -940,7 +940,7 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, // skip if matched by '..' in name const char *suffix = pathname + pathlen; - size_t sufflen; + lfs_size_t sufflen; int depth = 1; while (true) { suffix += strspn(suffix, "/"); @@ -1019,6 +1019,142 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, } } +/// Internal attribute operations /// +static int lfs_dir_getinfo(lfs_t *lfs, + lfs_dir_t *dir, const lfs_entry_t *entry, struct lfs_info *info) { + memset(info, 0, sizeof(*info)); + info->type = 0xf & entry->d.type; + if (entry->d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { + info->size = entry->d.u.file.size; + } else if (entry->d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = lfs_entry_elen(entry); + } + + if (lfs_paircmp(entry->d.u.dir, lfs->root) == 0) { + strcpy(info->name, "/"); + } else { + int err = lfs_dir_get(lfs, dir, + entry->off + entry->size - entry->d.nlen, + info->name, entry->d.nlen); + if (err) { + return err; + } + } + + return 0; +} + +static int lfs_dir_getattr(lfs_t *lfs, + lfs_dir_t *dir, const lfs_entry_t *entry, + uint8_t type, void *buffer, lfs_size_t size) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + if (attr.d.len > size) { + return LFS_ERR_RANGE; + } + + err = lfs_dir_get(lfs, dir, + entry->off+off+i+sizeof(attr.d), buffer, attr.d.len); + if (err) { + return err; + } + + return attr.d.len; + } + + return LFS_ERR_NODATA; +} + +static int lfs_dir_setattr(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, + uint8_t type, const void *buffer, lfs_size_t size) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + lfs_size_t oldlen = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + oldlen = attr.d.len; + break; + } + + // make sure the attribute fits + if (lfs_entry_elen(entry) - oldlen + size > lfs->attrs_size || + (0x7fffffff & dir->d.size) - oldlen + size > lfs->cfg->block_size) { + return LFS_ERR_NOSPC; + } + + lfs_attr_t attr; + attr.d.type = type; + attr.d.len = size; + int err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_MEM, off+i, &attr.d, sizeof(attr.d)}, + {LFS_FROM_MEM, off+i, buffer, size}, + {LFS_FROM_DROP, off+i, NULL, -oldlen}}, 3); + if (err) { + return err; + } + + return 0; +} + +static int lfs_dir_removeattr(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, uint8_t type) { + // search for attribute in attribute region + lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); + lfs_off_t i = 0; + while (i < lfs_entry_alen(entry)) { + lfs_attr_t attr; + int err = lfs_dir_get(lfs, dir, + entry->off+off+i, &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + if (attr.d.type != type) { + i += attr.d.len; + continue; + } + + err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_DROP, off+i, + NULL, -(sizeof(attr.d)+attr.d.len)}}, 1); + if (err) { + return err; + } + + return 0; + } + + return LFS_ERR_NODATA; +} + + /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { @@ -1179,16 +1315,7 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { break; } - info->type = 0xf & entry.d.type; - if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { - info->size = entry.d.u.file.size; - } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = lfs_entry_elen(&entry); - } - - int err = lfs_dir_get(lfs, dir, - entry.off + entry.size - entry.d.nlen, - info->name, entry.d.nlen); + int err = lfs_dir_getinfo(lfs, dir, &entry, info); if (err) { return err; } @@ -2048,26 +2175,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { return err; } - memset(info, 0, sizeof(*info)); - info->type = 0xf & entry.d.type; - if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { - info->size = entry.d.u.file.size; - } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { - info->size = lfs_entry_elen(&entry); - } - - if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) { - strcpy(info->name, "/"); - } else { - err = lfs_dir_get(lfs, &cwd, - entry.off + entry.size - entry.d.nlen, - info->name, entry.d.nlen); - if (err) { - return err; - } - } - - return 0; + return lfs_dir_getinfo(lfs, &cwd, &entry, info); } int lfs_remove(lfs_t *lfs, const char *path) { @@ -2263,6 +2371,58 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } +/// Attribute operations /// +int lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_getattr(lfs, &cwd, &entry, type, buffer, size); +} + +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_setattr(lfs, &cwd, &entry, type, buffer, size); +} + +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, lfs->root); + if (err) { + return err; + } + + lfs_entry_t entry; + err = lfs_dir_find(lfs, &cwd, &entry, &path); + if (err) { + return err; + } + + return lfs_dir_removeattr(lfs, &cwd, &entry, type); +} + + /// Filesystem operations /// static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->cfg = cfg; diff --git a/lfs.h b/lfs.h index 55148231..adc1c02d 100644 --- a/lfs.h +++ b/lfs.h @@ -89,6 +89,8 @@ enum lfs_error { LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available LFS_ERR_NAMETOOLONG = -36, // File name too long + LFS_ERR_NODATA = -61, // No data/attr available + LFS_ERR_RANGE = -34, // Result not representable }; // File types @@ -253,6 +255,13 @@ typedef struct lfs_entry { } d; } lfs_entry_t; +typedef struct lfs_attr { + struct lfs_disk_attr { + uint8_t type; + uint8_t len; + } d; +} lfs_attr_t; + typedef struct lfs_cache { lfs_block_t block; lfs_off_t off; @@ -379,6 +388,29 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); // Returns a negative error code on failure. int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); +// Get a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns the size of the attribute, or a negative error code on failure. +int lfs_getattr(lfs_t *lfs, const char *path, + uint8_t type, void *buffer, lfs_size_t size); + +// Set a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns a negative error code on failure. +int lfs_setattr(lfs_t *lfs, const char *path, + uint8_t type, const void *buffer, lfs_size_t size); + +// Remove a custom attribute +// +// Attributes are identified by an 8-bit type and are limited to at +// most LFS_ATTRS_SIZE bytes. +// Returns a negative error code on failure. +int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); + /// File operations /// From 6c754c802352d7c46b8b10ba4dd8088b2cea2d86 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 5 Apr 2018 23:23:14 -0500 Subject: [PATCH 23/31] Added support for atomically committing custom attributes Although it's simple and probably what most users expect, the previous custom attributes API suffered from one problem: the inability to update attributes atomically. If we consider our timestamp use case, updating a file would require: 1. Update the file 2. Update the timestamp If a power loss occurs during this sequence of updates, we could end up with a file with an incorrect timestamp. Is this a big deal? Probably not, but it could be a surprise only found after a power-loss. And littlefs was developed with the _specifically_ to avoid suprises during power-loss. The littlefs is perfectly capable of bundling multiple attribute updates in a single directory commit. That's kind of what it was designed to do. So all we need is a new committer opcode for list of attributes, and then poking that list of attributes through the API. We could provide the single-attribute functions, but don't, because the fewer functions makes for a smaller codebase, and these are already the more advanced functions so we can expect more from users. This also changes semantics about what happens when we don't find an attribute, since erroring would throw away all of the other attributes we're processing. To atomically commit both custom attributes and file updates, we need a new API, lfs_file_setattr. Unfortunately the semantics are a bit more confusing than lfs_setattr, since the attributes aren't written out immediately. --- lfs.c | 242 ++++++++++++++++++++++++++++++++++------------------------ lfs.h | 94 ++++++++++++++++++----- 2 files changed, 218 insertions(+), 118 deletions(-) diff --git a/lfs.c b/lfs.c index fa63e8ee..b079ef4a 100644 --- a/lfs.c +++ b/lfs.c @@ -515,6 +515,7 @@ struct lfs_region { LFS_FROM_DROP, LFS_FROM_MEM, LFS_FROM_REGION, + LFS_FROM_ATTRS, } type; lfs_off_t off; @@ -522,6 +523,12 @@ struct lfs_region { lfs_ssize_t size; }; +struct lfs_attrs_region { + const struct lfs_attr *attrs; + int count; + lfs_size_t len; +}; + struct lfs_region_region { lfs_block_t block; lfs_off_t off; @@ -568,6 +575,73 @@ static int lfs_commit_region(lfs_t *lfs, newoff += regions[i].size; break; } + case LFS_FROM_ATTRS: { + const struct lfs_attrs_region *attrs = regions[i].buffer; + + // order doesn't matter, so we write new attrs first. this + // is still O(n^2) but only O(n) disk access + for (int j = 0; j < attrs->count; j++) { + if (attrs->attrs[j].size == 0) { + continue; + } + + lfs_entry_attr_t attr = { + .d.type = attrs->attrs[j].type, + .d.len = attrs->attrs[j].size, + }; + + lfs_crc(crc, &attr.d, sizeof(attr.d)); + int err = lfs_bd_prog(lfs, newblock, newoff, + &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + lfs_crc(crc, + attrs->attrs[j].buffer, attrs->attrs[j].size); + err = lfs_bd_prog(lfs, newblock, newoff+sizeof(attr.d), + attrs->attrs[j].buffer, attrs->attrs[j].size); + if (err) { + return err; + } + + newoff += sizeof(attr.d) + attrs->attrs[j].size; + } + + // copy over attributes without updates + lfs_entry_attr_t attr; + for (lfs_off_t k = 0; k < attrs->len; k += 2+attr.d.len) { + int err = lfs_bd_read(lfs, oldblock, oldoff, + &attr.d, sizeof(attr.d)); + if (err) { + return err; + } + + bool updating = false; + for (int j = 0; j < attrs->count; j++) { + if (attr.d.type == attrs->attrs[j].type) { + updating = true; + } + } + + if (!updating) { + err = lfs_commit_region(lfs, + oldblock, oldoff, + newblock, newoff, + 0, NULL, 0, + attr.d.len, crc); + if (err) { + return err; + } + + newoff += 2+attr.d.len; + } + + oldoff += 2+attr.d.len; + } + + break; + } } i += 1; @@ -590,6 +664,8 @@ static int lfs_commit_region(lfs_t *lfs, } } + // sanity check our commit math + LFS_ASSERT(newoff == end); return 0; } @@ -1044,118 +1120,102 @@ static int lfs_dir_getinfo(lfs_t *lfs, return 0; } -static int lfs_dir_getattr(lfs_t *lfs, +static int lfs_dir_getattrs(lfs_t *lfs, lfs_dir_t *dir, const lfs_entry_t *entry, - uint8_t type, void *buffer, lfs_size_t size) { + const struct lfs_attr *attrs, int count) { + // set to zero in case we can't find the attributes or size mismatch + for (int j = 0; j < count; j++) { + memset(attrs[j].buffer, 0, attrs[j].size); + } + // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; + lfs_off_t off = 4+lfs_entry_elen(entry); + lfs_entry_attr_t attr; + for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { int err = lfs_dir_get(lfs, dir, entry->off+off+i, &attr.d, sizeof(attr.d)); if (err) { return err; } - if (attr.d.type != type) { - i += attr.d.len; - continue; - } - - if (attr.d.len > size) { - return LFS_ERR_RANGE; - } + for (int j = 0; j < count; j++) { + if (attr.d.type == attrs[j].type) { + if (attr.d.len > attrs[j].size) { + return LFS_ERR_RANGE; + } - err = lfs_dir_get(lfs, dir, - entry->off+off+i+sizeof(attr.d), buffer, attr.d.len); - if (err) { - return err; + err = lfs_dir_get(lfs, dir, + entry->off+off+i+sizeof(attr.d), + attrs[j].buffer, attr.d.len); + if (err) { + return err; + } + } } - - return attr.d.len; } - return LFS_ERR_NODATA; + return 0; } -static int lfs_dir_setattr(lfs_t *lfs, +static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, - uint8_t type, const void *buffer, lfs_size_t size) { - // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - lfs_size_t oldlen = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; + const struct lfs_attr *attrs, int count) { + // check that attributes fit + lfs_size_t nsize = 0; + for (int j = 0; j < count; j++) { + nsize += 2+attrs[j].size; + } + + lfs_off_t off = 4+lfs_entry_elen(entry); + lfs_entry_attr_t attr; + for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { int err = lfs_dir_get(lfs, dir, entry->off+off+i, &attr.d, sizeof(attr.d)); if (err) { return err; } - if (attr.d.type != type) { - i += attr.d.len; - continue; + bool updated = false; + for (int j = 0; j < count; j++) { + if (attr.d.type == attrs[j].type) { + updated = true; + } } - oldlen = attr.d.len; - break; + if (!updated) { + nsize += 2+attr.d.len; + } } - // make sure the attribute fits - if (lfs_entry_elen(entry) - oldlen + size > lfs->attrs_size || - (0x7fffffff & dir->d.size) - oldlen + size > lfs->cfg->block_size) { + if (nsize > lfs->attrs_size || ( + (0x7fffffff & dir->d.size) + lfs_entry_size(entry) - + lfs_entry_alen(entry) + nsize > lfs->cfg->block_size)) { return LFS_ERR_NOSPC; } - lfs_attr_t attr; - attr.d.type = type; - attr.d.len = size; - int err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_MEM, off+i, &attr.d, sizeof(attr.d)}, - {LFS_FROM_MEM, off+i, buffer, size}, - {LFS_FROM_DROP, off+i, NULL, -oldlen}}, 3); - if (err) { - return err; - } - - return 0; + return nsize; } -static int lfs_dir_removeattr(lfs_t *lfs, - lfs_dir_t *dir, lfs_entry_t *entry, uint8_t type) { - // search for attribute in attribute region - lfs_off_t off = sizeof(dir->d) + lfs_entry_elen(entry); - lfs_off_t i = 0; - while (i < lfs_entry_alen(entry)) { - lfs_attr_t attr; - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); - if (err) { - return err; - } - - if (attr.d.type != type) { - i += attr.d.len; - continue; - } - - err = lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_DROP, off+i, - NULL, -(sizeof(attr.d)+attr.d.len)}}, 1); - if (err) { - return err; - } - - return 0; +static int lfs_dir_setattrs(lfs_t *lfs, + lfs_dir_t *dir, lfs_entry_t *entry, + const struct lfs_attr *attrs, int count) { + // make sure attributes fit + lfs_ssize_t nsize = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); + if (nsize < 0) { + return nsize; } - return LFS_ERR_NODATA; + // commit to entry, majority of work is in LFS_FROM_ATTRS + lfs_size_t oldlen = lfs_entry_alen(entry); + entry->d.alen = (0xc0 & entry->d.alen) | nsize; + return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, &entry->d, 4}, + {LFS_FROM_DROP, 0, NULL, -4}, + {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), + &(struct lfs_attrs_region){attrs, count, oldlen}, nsize}}, 3); } - /// Top level directory operations /// int lfs_mkdir(lfs_t *lfs, const char *path) { // deorphan if we haven't yet, needed at most once after poweron @@ -2372,25 +2432,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { /// Attribute operations /// -int lfs_getattr(lfs_t *lfs, const char *path, - uint8_t type, void *buffer, lfs_size_t size) { - lfs_dir_t cwd; - int err = lfs_dir_fetch(lfs, &cwd, lfs->root); - if (err) { - return err; - } - - lfs_entry_t entry; - err = lfs_dir_find(lfs, &cwd, &entry, &path); - if (err) { - return err; - } - - return lfs_dir_getattr(lfs, &cwd, &entry, type, buffer, size); -} - -int lfs_setattr(lfs_t *lfs, const char *path, - uint8_t type, const void *buffer, lfs_size_t size) { +int lfs_getattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { @@ -2403,10 +2446,11 @@ int lfs_setattr(lfs_t *lfs, const char *path, return err; } - return lfs_dir_setattr(lfs, &cwd, &entry, type, buffer, size); + return lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); } -int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { +int lfs_setattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; int err = lfs_dir_fetch(lfs, &cwd, lfs->root); if (err) { @@ -2419,7 +2463,7 @@ int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) { return err; } - return lfs_dir_removeattr(lfs, &cwd, &entry, type); + return lfs_dir_setattrs(lfs, &cwd, &entry, attrs, count); } diff --git a/lfs.h b/lfs.h index adc1c02d..9de7a3a6 100644 --- a/lfs.h +++ b/lfs.h @@ -234,6 +234,18 @@ struct lfs_info { char name[LFS_NAME_MAX+1]; }; +// Custom attribute structure +struct lfs_attr { + // Type of attribute, provided by user and used to identify the attribute + uint8_t type; + + // Pointer to buffer containing the attribute + void *buffer; + + // Size of attribute in bytes, limited to LFS_ATTRS_MAX + lfs_size_t size; +}; + /// littlefs data structures /// typedef struct lfs_entry { @@ -255,12 +267,12 @@ typedef struct lfs_entry { } d; } lfs_entry_t; -typedef struct lfs_attr { - struct lfs_disk_attr { +typedef struct lfs_entry_attr { + struct lfs_disk_entry_attr { uint8_t type; uint8_t len; } d; -} lfs_attr_t; +} lfs_entry_attr_t; typedef struct lfs_cache { lfs_block_t block; @@ -388,28 +400,25 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath); // Returns a negative error code on failure. int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info); -// Get a custom attribute +// Get custom attributes // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. -// Returns the size of the attribute, or a negative error code on failure. -int lfs_getattr(lfs_t *lfs, const char *path, - uint8_t type, void *buffer, lfs_size_t size); - -// Set a custom attribute +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. // Returns a negative error code on failure. -int lfs_setattr(lfs_t *lfs, const char *path, - uint8_t type, const void *buffer, lfs_size_t size); +int lfs_getattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count); -// Remove a custom attribute +// Set custom attributes +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. // -// Attributes are identified by an 8-bit type and are limited to at -// most LFS_ATTRS_SIZE bytes. // Returns a negative error code on failure. -int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type); +int lfs_setattrs(lfs_t *lfs, const char *path, + const struct lfs_attr *attrs, int count); /// File operations /// @@ -484,6 +493,30 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file); // Returns the size of the file, or a negative error code on failure. lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); +// Get custom attributes attached to a file +// +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. +// +// Returns a negative error code on failure. +int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count); + +// Set custom attributes on a file +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. +// +// Note: Attributes are not written out until a call to lfs_file_sync +// or lfs_file_close and must be allocated until the file is closed or +// lfs_file_setattrs is called with a count of zero. +// +// Returns a negative error code on failure. +int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count); + /// Directory operations /// @@ -532,6 +565,29 @@ lfs_soff_t lfs_dir_tell(lfs_t *lfs, lfs_dir_t *dir); int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); +/// Filesystem filesystem operations /// + +// Get custom attributes on the filesystem +// +// Attributes are looked up based on the type id. If the stored attribute is +// smaller than the buffer, it is padded with zeros. It the stored attribute +// is larger than the buffer, LFS_ERR_RANGE is returned. +// +// Returns a negative error code on failure. +int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); + +// Set custom attributes on the filesystem +// +// The array of attributes will be used to update the attributes stored on +// disk based on their type id. Unspecified attributes are left unmodified. +// Specifying an attribute with zero size deletes the attribute. +// +// Note: Filesystem level attributes are not available for wear-leveling +// +// Returns a negative error code on failure. +int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); + + /// Miscellaneous littlefs specific operations /// // Traverse through all blocks in use by the filesystem From 636c0ed3d1952cc2830e7ed695ebc492f84768d6 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 04:23:23 -0500 Subject: [PATCH 24/31] Modified commit regions to work better with custom attributes Mostly just removed LFS_FROM_DROP and changed the DSL grammar a bit to allow drops to occur naturally through oldsize -> newsize diff expressed in the region struct. This prevents us from having to add a drop every time we want to update an entry in-place. --- lfs.c | 196 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 100 deletions(-) diff --git a/lfs.c b/lfs.c index b079ef4a..e68d37f1 100644 --- a/lfs.c +++ b/lfs.c @@ -512,21 +512,20 @@ static int lfs_dir_fetch(lfs_t *lfs, struct lfs_region { enum { - LFS_FROM_DROP, LFS_FROM_MEM, LFS_FROM_REGION, LFS_FROM_ATTRS, } type; - lfs_off_t off; + lfs_off_t oldoff; + lfs_size_t oldsize; const void *buffer; - lfs_ssize_t size; + lfs_size_t newsize; }; -struct lfs_attrs_region { +struct lfs_region_attrs { const struct lfs_attr *attrs; int count; - lfs_size_t len; }; struct lfs_region_region { @@ -536,47 +535,44 @@ struct lfs_region_region { int count; }; -static int lfs_commit_region(lfs_t *lfs, +static int lfs_commit_region(lfs_t *lfs, uint32_t *crc, lfs_block_t oldblock, lfs_off_t oldoff, lfs_block_t newblock, lfs_off_t newoff, - lfs_off_t regionoff, - const struct lfs_region *regions, int count, - lfs_size_t size, uint32_t *crc) { + lfs_off_t regionoff, lfs_size_t regionsize, + const struct lfs_region *regions, int count) { int i = 0; - lfs_size_t end = newoff + size; - while (newoff < end) { + lfs_size_t newend = newoff + regionsize; + while (newoff < newend) { // commit from different types of regions - if (i < count && regions[i].off == oldoff - regionoff) { + if (i < count && regions[i].oldoff == oldoff - regionoff) { switch (regions[i].type) { - case LFS_FROM_DROP: { - oldoff -= regions[i].size; - break; - } case LFS_FROM_MEM: { - lfs_crc(crc, regions[i].buffer, regions[i].size); + lfs_crc(crc, regions[i].buffer, regions[i].newsize); int err = lfs_bd_prog(lfs, newblock, newoff, - regions[i].buffer, regions[i].size); + regions[i].buffer, regions[i].newsize); if (err) { return err; } - newoff += regions[i].size; + newoff += regions[i].newsize; + oldoff += regions[i].oldsize; break; } case LFS_FROM_REGION: { const struct lfs_region_region *disk = regions[i].buffer; - int err = lfs_commit_region(lfs, + int err = lfs_commit_region(lfs, crc, disk->block, disk->off, newblock, newoff, - disk->off, disk->regions, disk->count, - regions[i].size, crc); + disk->off, regions[i].newsize, + disk->regions, disk->count); if (err) { return err; } - newoff += regions[i].size; + newoff += regions[i].newsize; + oldoff -= regions[i].oldsize; break; } case LFS_FROM_ATTRS: { - const struct lfs_attrs_region *attrs = regions[i].buffer; + const struct lfs_region_attrs *attrs = regions[i].buffer; // order doesn't matter, so we write new attrs first. this // is still O(n^2) but only O(n) disk access @@ -585,10 +581,9 @@ static int lfs_commit_region(lfs_t *lfs, continue; } - lfs_entry_attr_t attr = { - .d.type = attrs->attrs[j].type, - .d.len = attrs->attrs[j].size, - }; + lfs_entry_attr_t attr; + attr.d.type = attrs->attrs[j].type; + attr.d.len = attrs->attrs[j].size; lfs_crc(crc, &attr.d, sizeof(attr.d)); int err = lfs_bd_prog(lfs, newblock, newoff, @@ -605,12 +600,13 @@ static int lfs_commit_region(lfs_t *lfs, return err; } - newoff += sizeof(attr.d) + attrs->attrs[j].size; + newoff += 2+attrs->attrs[j].size; } // copy over attributes without updates - lfs_entry_attr_t attr; - for (lfs_off_t k = 0; k < attrs->len; k += 2+attr.d.len) { + lfs_off_t oldend = oldoff + regions[i].oldsize; + while (oldoff < oldend) { + lfs_entry_attr_t attr; int err = lfs_bd_read(lfs, oldblock, oldoff, &attr.d, sizeof(attr.d)); if (err) { @@ -625,11 +621,11 @@ static int lfs_commit_region(lfs_t *lfs, } if (!updating) { - err = lfs_commit_region(lfs, + err = lfs_commit_region(lfs, crc, oldblock, oldoff, newblock, newoff, - 0, NULL, 0, - attr.d.len, crc); + 0, 2+attr.d.len, + NULL, 0); if (err) { return err; } @@ -665,7 +661,7 @@ static int lfs_commit_region(lfs_t *lfs, } // sanity check our commit math - LFS_ASSERT(newoff == end); + LFS_ASSERT(newoff == newend); return 0; } @@ -681,7 +677,8 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, // keep pairs in order such that pair[0] is most recent lfs_pairswap(dir->pair); for (int i = 0; i < count; i++) { - dir->d.size += regions[i].size; + dir->d.size += regions[i].newsize; + dir->d.size -= regions[i].oldsize; } while (true) { @@ -708,12 +705,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } // commit region - err = lfs_commit_region(lfs, + err = lfs_commit_region(lfs, &crc, dir->pair[1], sizeof(dir->d), dir->pair[0], sizeof(dir->d), - 0, regions, count, - (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, - &crc); + 0, (0x7fffffff & dir->d.size)-sizeof(dir->d)-4, + regions, count); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -807,7 +803,8 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions, int count) { lfs_ssize_t diff = 0; for (int i = 0; i < count; i++) { - diff += regions[i].size; + diff += regions[i].newsize; + diff -= regions[i].oldsize; } lfs_size_t oldsize = entry->size; @@ -829,8 +826,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, type |= LFS_STRUCT_MOVED; err = lfs_dir_commit(lfs, &olddir, (struct lfs_region[]){ - {LFS_FROM_MEM, oldoff, &type, 1}, - {LFS_FROM_DROP, oldoff, NULL, -1}}, 2); + {LFS_FROM_MEM, oldoff, 1, &type, 1}}, 1); if (err) { return err; } @@ -865,7 +861,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, entry->off = dir->d.size - 4; entry->size += diff; int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ - {LFS_FROM_REGION, entry->off, &(struct lfs_region_region){ + {LFS_FROM_REGION, entry->off, 0, &(struct lfs_region_region){ olddir.pair[0], oldoff, regions, count}, entry->size}}, 1); if (err) { @@ -893,7 +889,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, lfs_entry_t oldentry; oldentry.off = oldoff; err = lfs_dir_set(lfs, &olddir, &oldentry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -oldsize}}, 1); + {LFS_FROM_MEM, 0, oldsize, NULL, 0}}, 1); if (err) { return err; } @@ -922,7 +918,7 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, } for (int i = 0; i < count; i++) { - regions[i].off += entry->off; + regions[i].oldoff += entry->off; } int err = lfs_dir_commit(lfs, dir, regions, count); @@ -1129,11 +1125,11 @@ static int lfs_dir_getattrs(lfs_t *lfs, } // search for attribute in attribute region - lfs_off_t off = 4+lfs_entry_elen(entry); - lfs_entry_attr_t attr; - for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); + lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); + lfs_off_t end = off + lfs_entry_alen(entry); + while (off < end) { + lfs_entry_attr_t attr; + int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); if (err) { return err; } @@ -1144,14 +1140,15 @@ static int lfs_dir_getattrs(lfs_t *lfs, return LFS_ERR_RANGE; } - err = lfs_dir_get(lfs, dir, - entry->off+off+i+sizeof(attr.d), + err = lfs_dir_get(lfs, dir, off+sizeof(attr.d), attrs[j].buffer, attr.d.len); if (err) { return err; } } } + + off += 2+attr.d.len; } return 0; @@ -1161,16 +1158,19 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const struct lfs_attr *attrs, int count) { // check that attributes fit + // two separate passes so disk access is O(n) lfs_size_t nsize = 0; for (int j = 0; j < count; j++) { - nsize += 2+attrs[j].size; + if (attrs[j].size > 0) { + nsize += 2+attrs[j].size; + } } - lfs_off_t off = 4+lfs_entry_elen(entry); - lfs_entry_attr_t attr; - for (lfs_off_t i = 0; i < lfs_entry_alen(entry); i += 2+attr.d.len) { - int err = lfs_dir_get(lfs, dir, - entry->off+off+i, &attr.d, sizeof(attr.d)); + lfs_off_t off = entry->off + 4+lfs_entry_elen(entry); + lfs_off_t end = off + lfs_entry_alen(entry); + while (off < end) { + lfs_entry_attr_t attr; + int err = lfs_dir_get(lfs, dir, off, &attr.d, sizeof(attr.d)); if (err) { return err; } @@ -1185,6 +1185,8 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, if (!updated) { nsize += 2+attr.d.len; } + + off += 2+attr.d.len; } if (nsize > lfs->attrs_size || ( @@ -1200,19 +1202,18 @@ static int lfs_dir_setattrs(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, const struct lfs_attr *attrs, int count) { // make sure attributes fit - lfs_ssize_t nsize = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); - if (nsize < 0) { - return nsize; + lfs_size_t oldlen = lfs_entry_alen(entry); + lfs_ssize_t newlen = lfs_dir_checkattrs(lfs, dir, entry, attrs, count); + if (newlen < 0) { + return newlen; } // commit to entry, majority of work is in LFS_FROM_ATTRS - lfs_size_t oldlen = lfs_entry_alen(entry); - entry->d.alen = (0xc0 & entry->d.alen) | nsize; + entry->d.alen = (0xc0 & entry->d.alen) | newlen; return lfs_dir_set(lfs, dir, entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry->d, 4}, - {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), - &(struct lfs_attrs_region){attrs, count, oldlen}, nsize}}, 3); + {LFS_FROM_MEM, 0, 4, &entry->d, 4}, + {LFS_FROM_ATTRS, 4+lfs_entry_elen(entry), oldlen, + &(struct lfs_region_attrs){attrs, count}, newlen}}, 2); } @@ -1272,8 +1273,8 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_MEM, 0, path, nlen}}, 2); + {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, + {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); if (err) { return err; } @@ -1657,8 +1658,8 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, entry.size = 0; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, path, nlen}}, 2); + {LFS_FROM_MEM, 0, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); if (err) { return err; } @@ -1910,8 +1911,8 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.size = file->size; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 2); + {LFS_FROM_MEM, 0, 4+oldlen, + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } @@ -1921,9 +1922,9 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, file->cache.buffer, file->size}, - {LFS_FROM_DROP, 0, NULL, -oldlen-4}}, 3); + {LFS_FROM_MEM, 0, 0, &entry.d, 4}, + {LFS_FROM_MEM, 0, 4+oldlen, + file->cache.buffer, file->size}}, 2); if (err) { return err; } @@ -2274,7 +2275,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { // remove the entry err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); + {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); if (err) { return err; } @@ -2364,8 +2365,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // mark as moving oldentry.d.type |= LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &oldentry.d.type, 1}, - {LFS_FROM_DROP, 0, NULL, -1}}, 2); + {LFS_FROM_MEM, 0, 1, &oldentry.d.type, 1}}, 1); oldentry.d.type &= ~LFS_STRUCT_MOVED; if (err) { return err; @@ -2381,19 +2381,16 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { newentry.d = oldentry.d; newentry.d.type &= ~LFS_STRUCT_MOVED; newentry.d.nlen = nlen; - if (!prevexists) { - newentry.size = 0; - } + newentry.size = prevexists ? preventry.size : 0; lfs_size_t newsize = oldentry.size - oldentry.d.nlen + newentry.d.nlen; err = lfs_dir_set(lfs, &newcwd, &newentry, (struct lfs_region[]){ - {LFS_FROM_REGION, 0, &(struct lfs_region_region){ - oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &newentry.d, 4}, - {LFS_FROM_DROP, 0, NULL, -4}, - {LFS_FROM_MEM, newsize - nlen, newpath, nlen}}, 3}, - newsize}, - {LFS_FROM_DROP, 0, NULL, -preventry.size}}, prevexists ? 2 : 1); + {LFS_FROM_REGION, 0, prevexists ? preventry.size : 0, + &(struct lfs_region_region){ + oldcwd.pair[0], oldentry.off, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, 4, &newentry.d, 4}, + {LFS_FROM_MEM, newsize-nlen, 0, newpath, nlen}}, 2}, + newsize}}, 1); if (err) { return err; } @@ -2405,7 +2402,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // remove old entry err = lfs_dir_set(lfs, &oldcwd, &oldentry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -oldentry.size}}, 1); + {LFS_FROM_MEM, 0, oldentry.size, NULL, 0}}, 1); if (err) { return err; } @@ -2619,9 +2616,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { lfs_entry_tole32(&superentry.d); lfs_superblock_tole32(&superblock.d); err = lfs_dir_set(lfs, &superdir, &superentry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &superentry.d, 4}, - {LFS_FROM_MEM, 0, &superblock.d, sizeof(superblock.d)}, - {LFS_FROM_MEM, 0, "littlefs", superentry.d.nlen}}, 3); + {LFS_FROM_MEM, 0, 0, &superentry.d, 4}, + {LFS_FROM_MEM, 0, 0, &superblock.d, sizeof(superblock.d)}, + {LFS_FROM_MEM, 0, 0, "littlefs", superentry.d.nlen}}, 3); if (err) { return err; } @@ -2925,8 +2922,8 @@ static int lfs_relocate(lfs_t *lfs, entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, (lfs_ssize_t)-sizeof(entry.d)}}, 2); + {LFS_FROM_MEM, 0, sizeof(entry.d), + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } @@ -3043,7 +3040,7 @@ int lfs_deorphan(lfs_t *lfs) { LFS_DEBUG("Found move %d %d", entry.d.u.dir[0], entry.d.u.dir[1]); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_DROP, 0, NULL, -entry.size}}, 1); + {LFS_FROM_MEM, 0, entry.size, NULL, 0}}, 1); if (err) { return err; } @@ -3052,9 +3049,8 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, &entry.d, sizeof(entry.d)}, - {LFS_FROM_DROP, 0, NULL, - (lfs_ssize_t)-sizeof(entry.d)}}, 2); + {LFS_FROM_MEM, 0, sizeof(entry.d), + &entry.d, sizeof(entry.d)}}, 1); if (err) { return err; } From 93244a3734df6bea67a3cbf7b0d8519859e6b434 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 16:58:12 -0500 Subject: [PATCH 25/31] Added file-level and fs-level custom attribute APIs In the form of lfs_file_setattr, lfs_file_getattr, lfs_fs_setattr, lfs_fs_getattr. This enables atomic updates of custom attributes as described in 6c754c8, and provides a custom attribute API that allows custom attributes to be stored on the filesystem itself. --- lfs.c | 307 ++++++++++++++++++++++++++++++++++++++++------------------ lfs.h | 5 +- 2 files changed, 217 insertions(+), 95 deletions(-) diff --git a/lfs.c b/lfs.c index e68d37f1..efc8744c 100644 --- a/lfs.c +++ b/lfs.c @@ -801,7 +801,7 @@ static int lfs_dir_get(lfs_t *lfs, const lfs_dir_t *dir, static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, struct lfs_region *regions, int count) { - lfs_ssize_t diff = 0; + lfs_ssize_t diff = 0; for (int i = 0; i < count; i++) { diff += regions[i].newsize; diff -= regions[i].oldsize; @@ -932,11 +932,11 @@ static int lfs_dir_set(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry, // shift over any files/directories that are affected for (lfs_file_t *f = lfs->files; f; f = f->next) { if (lfs_paircmp(f->pair, dir->pair) == 0) { - if (f->poff == entry->off && entry->size == 0) { + if (f->pairoff == entry->off && entry->size == 0) { f->pair[0] = 0xffffffff; f->pair[1] = 0xffffffff; - } else if (f->poff > entry->off) { - f->poff += diff; + } else if (f->pairoff > entry->off) { + f->pairoff += diff; } } } @@ -1135,8 +1135,8 @@ static int lfs_dir_getattrs(lfs_t *lfs, } for (int j = 0; j < count; j++) { - if (attr.d.type == attrs[j].type) { - if (attr.d.len > attrs[j].size) { + if (attrs[j].type == attr.d.type) { + if (attrs[j].size < attr.d.len) { return LFS_ERR_RANGE; } @@ -1190,8 +1190,8 @@ static lfs_ssize_t lfs_dir_checkattrs(lfs_t *lfs, } if (nsize > lfs->attrs_size || ( - (0x7fffffff & dir->d.size) + lfs_entry_size(entry) - - lfs_entry_alen(entry) + nsize > lfs->cfg->block_size)) { + lfs_entry_size(entry) - lfs_entry_alen(entry) + nsize + > lfs->cfg->block_size)) { return LFS_ERR_NOSPC; } @@ -1688,7 +1688,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, // setup file struct file->pair[0] = cwd.pair[0]; file->pair[1] = cwd.pair[1]; - file->poff = entry.off; + file->pairoff = entry.off; file->flags = flags; file->pos = 0; @@ -1738,87 +1738,87 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_sync(lfs, file); + int err = lfs_file_sync(lfs, file); - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; - break; + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; + } } - } - - // clean up memory - if (!lfs->cfg->file_buffer) { - lfs_free(file->cache.buffer); - } - - return err; -} -static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { -relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); - if (err) { - return err; - } - - err = lfs_bd_erase(lfs, nblock); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; + // clean up memory + if (!lfs->cfg->file_buffer) { + lfs_free(file->cache.buffer); } + return err; } - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); + static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { + relocate:; + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); if (err) { return err; } - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); + err = lfs_bd_erase(lfs, nblock); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } - } - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); + if (err) { + return err; + } - file->block = nblock; - return 0; -} + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + } -static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; + + file->block = nblock; + return 0; } - if (file->flags & LFS_F_WRITING) { - lfs_off_t pos = file->pos; + static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } - if (!(file->flags & LFS_F_INLINE)) { - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; + + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; lfs->rcache.block = 0xffffffff; while (file->pos < file->size) { @@ -1892,42 +1892,54 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - lfs_entry_t entry = {.off = file->poff}; + lfs_entry_t entry = {.off = file->pairoff}; err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); lfs_entry_fromle32(&entry.d); if (err) { return err; } + entry.size = lfs_entry_size(&entry); LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); - lfs_size_t oldlen = lfs_entry_elen(&entry); - entry.size = lfs_entry_size(&entry); + lfs_size_t oldelen = lfs_entry_elen(&entry); + lfs_size_t oldalen = lfs_entry_alen(&entry); + const void *buffer; + lfs_size_t size; // either update the references or inline the whole file if (!(file->flags & LFS_F_INLINE)) { entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d)-4; entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 4+oldlen, - &entry.d, sizeof(entry.d)}}, 1); - if (err) { - return err; - } + buffer = (const uint8_t *)&entry.d + 4; + size = sizeof(entry.d) - 4; } else { entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; - entry.d.elen = file->size & 0xff; - entry.d.alen = (entry.d.alen & 0x3f) | ((file->size >> 2) & 0xc0); - err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, 0, &entry.d, 4}, - {LFS_FROM_MEM, 0, 4+oldlen, - file->cache.buffer, file->size}}, 2); - if (err) { - return err; - } + buffer = file->cache.buffer; + size = file->size; + } + + // get new alen from disk + lfs_ssize_t newalen = lfs_dir_checkattrs(lfs, &cwd, &entry, + file->attrs, file->attrcount); + if (newalen < 0) { + return newalen; + } + + entry.d.elen = size & 0xff; + entry.d.alen = (newalen & 0x3f) | ((size >> 2) & 0xc0); + + // write out update + err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ + {LFS_FROM_MEM, 0, 4, &entry.d, 4}, + {LFS_FROM_MEM, 4, oldelen, buffer, size}, + {LFS_FROM_ATTRS, 4+oldelen, oldalen, + &(struct lfs_region_attrs){file->attrs, file->attrcount}, + newalen}}, 3); + if (err) { + return err; } file->flags &= ~LFS_F_DIRTY; @@ -2221,6 +2233,81 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) { } } +int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count) { + // set to null in case we can't find the attrs (missing file?) + for (int j = 0; j < count; j++) { + memset(attrs[j].buffer, 0, attrs[j].size); + } + + // load from disk if we haven't already been deleted + if (!lfs_pairisnull(file->pair)) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, file->pair); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = file->pairoff}; + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + err = lfs_dir_getattrs(lfs, &cwd, &entry, attrs, count); + if (err) { + return err; + } + } + + // override an attrs we have stored locally + for (int i = 0; i < file->attrcount; i++) { + for (int j = 0; j < count; j++) { + if (attrs[j].type == file->attrs[i].type) { + if (attrs[j].size < file->attrs[i].size) { + return LFS_ERR_RANGE; + } + + memcpy(attrs[j].buffer, + file->attrs[i].buffer, file->attrs[i].size); + } + } + } + + return 0; +} + +int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, + const struct lfs_attr *attrs, int count) { + // just tack to the file, will be written at sync time + file->attrs = attrs; + file->attrcount = count; + + // at least make sure attributes fit + if (!lfs_pairisnull(file->pair)) { + lfs_dir_t cwd; + int err = lfs_dir_fetch(lfs, &cwd, file->pair); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = file->pairoff}; + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + lfs_ssize_t res = lfs_dir_checkattrs(lfs, &cwd, &entry, attrs, count); + if (res < 0) { + return res; + } + } + + return 0; +} + /// General fs operations /// int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) { @@ -2427,8 +2514,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return 0; } - -/// Attribute operations /// int lfs_getattrs(lfs_t *lfs, const char *path, const struct lfs_attr *attrs, int count) { lfs_dir_t cwd; @@ -2646,10 +2731,6 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { // load superblock lfs_dir_t dir; - lfs_entry_t entry; - lfs_superblock_t superblock; - char magic[8]; - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -2658,11 +2739,13 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - err = lfs_dir_get(lfs, &dir, sizeof(dir.d), &entry.d, sizeof(entry.d)); + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); if (err) { return err; } + lfs_superblock_t superblock; memset(&superblock.d, 0, sizeof(superblock.d)); err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+4, &superblock.d, @@ -2672,6 +2755,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } + char magic[8]; err = lfs_dir_get(lfs, &dir, sizeof(dir.d)+lfs_entry_size(&entry)-entry.d.nlen, magic, lfs_min(sizeof(magic), entry.d.nlen)); @@ -2733,7 +2817,7 @@ int lfs_unmount(lfs_t *lfs) { } -/// Littlefs specific operations /// +/// Internal filesystem filesystem operations /// int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { if (lfs_pairisnull(lfs->root)) { return 0; @@ -3064,3 +3148,38 @@ int lfs_deorphan(lfs_t *lfs) { return 0; } + +/// External filesystem filesystem operations /// +int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { + lfs_dir_t dir; + int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + return lfs_dir_getattrs(lfs, &dir, &entry, attrs, count); +} + +int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { + lfs_dir_t dir; + int err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err) { + return err; + } + + lfs_entry_t entry = {.off = sizeof(dir.d)}; + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + if (err) { + return err; + } + entry.size = lfs_entry_size(&entry); + + return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); +} diff --git a/lfs.h b/lfs.h index 9de7a3a6..1e81a38b 100644 --- a/lfs.h +++ b/lfs.h @@ -283,7 +283,7 @@ typedef struct lfs_cache { typedef struct lfs_file { struct lfs_file *next; lfs_block_t pair[2]; - lfs_off_t poff; + lfs_off_t pairoff; lfs_block_t head; lfs_size_t size; @@ -294,6 +294,9 @@ typedef struct lfs_file { lfs_block_t block; lfs_off_t off; lfs_cache_t cache; + + const struct lfs_attr *attrs; + int attrcount; } lfs_file_t; typedef struct lfs_dir { From 746b90965cfd5d81b91af6d6a63362adf0e08060 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 8 Apr 2018 22:25:58 -0500 Subject: [PATCH 26/31] Added lfs_fs_size for finding a count of used blocks This has existed for some time in the form of the lfs_traverse function, through which a user could provide a simple callback that would just count the number of blocks lfs_traverse finds. However, this approach is relatively unconventional and has proven to be confusing for most users. --- lfs.c | 16 ++++++++++++++++ lfs.h | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/lfs.c b/lfs.c index efc8744c..12c903f7 100644 --- a/lfs.c +++ b/lfs.c @@ -3183,3 +3183,19 @@ int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { return lfs_dir_setattrs(lfs, &dir, &entry, attrs, count); } + +static int lfs_fs_size_count(void *p, lfs_block_t block) { + lfs_size_t *size = p; + *size += 1; + return 0; +} + +lfs_ssize_t lfs_fs_size(lfs_t *lfs) { + lfs_size_t size = 0; + int err = lfs_traverse(lfs, lfs_fs_size_count, &size); + if (err) { + return err; + } + + return size; +} diff --git a/lfs.h b/lfs.h index 1e81a38b..3fd3ba13 100644 --- a/lfs.h +++ b/lfs.h @@ -590,6 +590,14 @@ int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); // Returns a negative error code on failure. int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count); +// Finds the current size of the filesystem +// +// Note: Result is best effort. If files share COW structures, the returned +// size may be larger than the filesystem actually is. +// +// Returns the number of allocated blocks, or a negative error code on failure. +lfs_ssize_t lfs_fs_size(lfs_t *lfs); + /// Miscellaneous littlefs specific operations /// From 2a8277bd4dafdec1d92f69e7992a339a45088c42 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 16:15:37 -0500 Subject: [PATCH 27/31] Added test coverage for filesystems with no inline files --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5c0988b5..54a5771e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ script: - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" + - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0" - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS" # compile and find the code size with the smallest configuration From ea4ded420ced77503f51917b41cdd157384f4eb4 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 16:35:29 -0500 Subject: [PATCH 28/31] Fixed big-endian support again This is what I get for not runing CI on a local development branch. --- lfs.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lfs.c b/lfs.c index 12c903f7..627b9d8c 100644 --- a/lfs.c +++ b/lfs.c @@ -1272,6 +1272,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) { cwd.d.tail[0] = dir.pair[0]; cwd.d.tail[1] = dir.pair[1]; + lfs_entry_tole32(&entry.d); err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, 0, &entry.d, sizeof(entry.d)}, {LFS_FROM_MEM, 0, 0, path, nlen}}, 2); @@ -1893,8 +1894,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); - lfs_entry_fromle32(&entry.d); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -1912,6 +1912,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; + lfs_entry_tole32(&entry.d); buffer = (const uint8_t *)&entry.d + 4; size = sizeof(entry.d) - 4; } else { @@ -2249,7 +2250,7 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -2293,7 +2294,7 @@ int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, } lfs_entry_t entry = {.off = file->pairoff}; - err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &cwd, entry.off, &entry.d, 4); if (err) { return err; } @@ -2740,7 +2741,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } @@ -3005,6 +3006,7 @@ static int lfs_relocate(lfs_t *lfs, // update disk, this creates a desync entry.d.u.dir[0] = newpair[0]; entry.d.u.dir[1] = newpair[1]; + lfs_entry_tole32(&entry.d); int err = lfs_dir_set(lfs, &parent, &entry, (struct lfs_region[]){ {LFS_FROM_MEM, 0, sizeof(entry.d), &entry.d, sizeof(entry.d)}}, 1); @@ -3133,8 +3135,7 @@ int lfs_deorphan(lfs_t *lfs) { entry.d.u.dir[0], entry.d.u.dir[1]); entry.d.type &= ~LFS_STRUCT_MOVED; err = lfs_dir_set(lfs, &cwd, &entry, (struct lfs_region[]){ - {LFS_FROM_MEM, 0, sizeof(entry.d), - &entry.d, sizeof(entry.d)}}, 1); + {LFS_FROM_MEM, 0, 1, &entry.d, 1}}, 1); if (err) { return err; } @@ -3158,7 +3159,7 @@ int lfs_fs_getattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } @@ -3175,7 +3176,7 @@ int lfs_fs_setattrs(lfs_t *lfs, const struct lfs_attr *attrs, int count) { } lfs_entry_t entry = {.off = sizeof(dir.d)}; - err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, sizeof(entry.d)); + err = lfs_dir_get(lfs, &dir, entry.off, &entry.d, 4); if (err) { return err; } From 61f454b008e8c8a7208e8b74095da8501d62c621 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Apr 2018 19:55:17 -0500 Subject: [PATCH 29/31] Added tests for resizable entries and custom attributes Also found some bugs. Should now have a good amount of confidence in these features. --- Makefile | 5 +- lfs.c | 132 ++++++++++--------- tests/test.py | 2 +- tests/test_attrs.sh | 292 ++++++++++++++++++++++++++++++++++++++++++ tests/test_entries.sh | 220 +++++++++++++++++++++++++++++++ tests/test_paths.sh | 18 +++ 6 files changed, 605 insertions(+), 64 deletions(-) create mode 100755 tests/test_attrs.sh create mode 100755 tests/test_entries.sh diff --git a/Makefile b/Makefile index c0933139..55c95dbd 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,9 @@ size: $(OBJ) $(SIZE) -t $^ .SUFFIXES: -test: test_format test_dirs test_files test_seek test_truncate \ - test_interspersed test_alloc test_paths test_orphan test_move test_corrupt +test: test_format test_dirs test_files test_seek test_truncate test_entries \ + test_interspersed test_alloc test_paths test_attrs \ + test_orphan test_move test_corrupt test_%: tests/test_%.sh ifdef QUIET @./$< | sed -n '/^[-=]/p' diff --git a/lfs.c b/lfs.c index 627b9d8c..49b4f3bc 100644 --- a/lfs.c +++ b/lfs.c @@ -1739,87 +1739,87 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { - int err = lfs_file_sync(lfs, file); + int err = lfs_file_sync(lfs, file); - // remove from list of files - for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { - if (*p == file) { - *p = file->next; - break; - } + // remove from list of files + for (lfs_file_t **p = &lfs->files; *p; p = &(*p)->next) { + if (*p == file) { + *p = file->next; + break; } + } - // clean up memory - if (!lfs->cfg->file_buffer) { - lfs_free(file->cache.buffer); - } + // clean up memory + if (!lfs->cfg->file_buffer) { + lfs_free(file->cache.buffer); + } + return err; +} + +static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { +relocate:; + // just relocate what exists into new block + lfs_block_t nblock; + int err = lfs_alloc(lfs, &nblock); + if (err) { return err; } - static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { - relocate:; - // just relocate what exists into new block - lfs_block_t nblock; - int err = lfs_alloc(lfs, &nblock); + err = lfs_bd_erase(lfs, nblock); + if (err) { + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } + return err; + } + + // either read from dirty cache or disk + for (lfs_off_t i = 0; i < file->off; i++) { + uint8_t data; + err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, + file->block, i, &data, 1); if (err) { return err; } - err = lfs_bd_erase(lfs, nblock); + err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, + nblock, i, &data, 1); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; } return err; } + } - // either read from dirty cache or disk - for (lfs_off_t i = 0; i < file->off; i++) { - uint8_t data; - err = lfs_cache_read(lfs, &lfs->rcache, &file->cache, - file->block, i, &data, 1); - if (err) { - return err; - } - - err = lfs_cache_prog(lfs, &lfs->pcache, &lfs->rcache, - nblock, i, &data, 1); - if (err) { - if (err == LFS_ERR_CORRUPT) { - goto relocate; - } - return err; - } - } + // copy over new state of file + memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); + file->cache.block = lfs->pcache.block; + file->cache.off = lfs->pcache.off; + lfs->pcache.block = 0xffffffff; - // copy over new state of file - memcpy(file->cache.buffer, lfs->pcache.buffer, lfs->cfg->prog_size); - file->cache.block = lfs->pcache.block; - file->cache.off = lfs->pcache.off; - lfs->pcache.block = 0xffffffff; + file->block = nblock; + return 0; +} - file->block = nblock; - return 0; +static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; } - static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { - if (file->flags & LFS_F_READING) { - file->flags &= ~LFS_F_READING; - } - - if (file->flags & LFS_F_WRITING) { - lfs_off_t pos = file->pos; + if (file->flags & LFS_F_WRITING) { + lfs_off_t pos = file->pos; - if (!(file->flags & LFS_F_INLINE)) { - // copy over anything after current branch - lfs_file_t orig = { - .head = file->head, - .size = file->size, - .flags = LFS_O_RDONLY, - .pos = file->pos, - .cache = lfs->rcache, - }; + if (!(file->flags & LFS_F_INLINE)) { + // copy over anything after current branch + lfs_file_t orig = { + .head = file->head, + .size = file->size, + .flags = LFS_O_RDONLY, + .pos = file->pos, + .cache = lfs->rcache, + }; lfs->rcache.block = 0xffffffff; while (file->pos < file->size) { @@ -2270,6 +2270,7 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, return LFS_ERR_RANGE; } + memset(attrs[j].buffer, 0, attrs[j].size); memcpy(attrs[j].buffer, file->attrs[i].buffer, file->attrs[i].size); } @@ -2281,9 +2282,9 @@ int lfs_file_getattrs(lfs_t *lfs, lfs_file_t *file, int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, const struct lfs_attr *attrs, int count) { - // just tack to the file, will be written at sync time - file->attrs = attrs; - file->attrcount = count; + if ((file->flags & 3) == LFS_O_RDONLY) { + return LFS_ERR_BADF; + } // at least make sure attributes fit if (!lfs_pairisnull(file->pair)) { @@ -2306,6 +2307,11 @@ int lfs_file_setattrs(lfs_t *lfs, lfs_file_t *file, } } + // just tack to the file, will be written at sync time + file->attrs = attrs; + file->attrcount = count; + file->flags |= LFS_F_DIRTY; + return 0; } @@ -2432,6 +2438,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return LFS_ERR_NAMETOOLONG; } + if (oldentry.size - oldentry.d.nlen + nlen > lfs->cfg->block_size) { + return LFS_ERR_NOSPC; + } + // must have same type if (prevexists && preventry.d.type != oldentry.d.type) { return LFS_ERR_ISDIR; diff --git a/tests/test.py b/tests/test.py index 24b0d1a3..e93ccec6 100755 --- a/tests/test.py +++ b/tests/test.py @@ -10,7 +10,7 @@ def generate(test): template = file.read() lines = [] - for line in re.split('(?<=[;{}])\n', test.read()): + for line in re.split('(?<=(?:.;| [{}]))\n', test.read()): match = re.match('(?: *\n)*( *)(.*)=>(.*);', line, re.DOTALL | re.MULTILINE) if match: tab, test, expect = match.groups() diff --git a/tests/test_attrs.sh b/tests/test_attrs.sh new file mode 100755 index 00000000..286b909d --- /dev/null +++ b/tests/test_attrs.sh @@ -0,0 +1,292 @@ +#!/bin/bash +set -eu + +echo "=== Attr tests ===" +rm -rf blocks +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + lfs_mkdir(&lfs, "hello") => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", + LFS_O_WRONLY | LFS_O_CREAT) => 0; + lfs_file_write(&lfs, &file[0], "hello", strlen("hello")) + => strlen("hello"); + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', "aaaa", 4}, + {'B', "bbbbbb", 6}, + {'C', "ccccc", 5}}, 3) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "", 0}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "dddddd", 6}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "eee", 3}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; + lfs_setattrs(&lfs, "hello", (struct lfs_attr[]){ + {'B', "fffffffff", 9}}, 1) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_getattrs(&lfs, "hello", (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get fs attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'A', "aaaa", 4}, + {'B', "bbbbbb", 6}, + {'C', "ccccc", 5}}, 3) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "", 0}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "dddddd", 6}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "eee", 3}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, LFS_ATTRS_MAX+1}}, 1) => LFS_ERR_NOSPC; + lfs_fs_setattrs(&lfs, (struct lfs_attr[]){ + {'B', "fffffffff", 9}}, 1) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_fs_getattrs(&lfs, (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Set/get file attribute ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_WRONLY) => 0; + + struct lfs_attr attr[3]; + attr[0] = (struct lfs_attr){'A', "aaaa", 4}; + attr[1] = (struct lfs_attr){'B', "bbbbbb", 6}; + attr[2] = (struct lfs_attr){'C', "ccccc", 5}; + lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "bbbbbb", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "", 0}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "\0\0\0\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "dddddd", 6}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "dddddd", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'B', "eee", 3}; + lfs_file_setattrs(&lfs, &file[0], attr, 1); + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "eee\0\0\0", 6) => 0; + memcmp(buffer+10, "ccccc", 5) => 0; + lfs_file_sync(&lfs, &file[0]) => 0; + + attr[0] = (struct lfs_attr){'A', buffer, LFS_ATTRS_MAX+1}; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => LFS_ERR_NOSPC; + attr[0] = (struct lfs_attr){'B', "fffffffff", 9}; + lfs_file_open(&lfs, &file[1], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_setattrs(&lfs, &file[1], attr, 1) => LFS_ERR_BADF; + lfs_file_close(&lfs, &file[1]) => 0; + lfs_file_setattrs(&lfs, &file[0], attr, 1) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 6}, + {'C', buffer+10, 5}}, 3) => LFS_ERR_RANGE; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'A', buffer, 4}, + {'B', buffer+4, 9}, + {'C', buffer+13, 5}}, 3) => 0; + memcmp(buffer, "aaaa", 4) => 0; + memcmp(buffer+4, "fffffffff", 9) => 0; + memcmp(buffer+13, "ccccc", 5) => 0; + + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], buffer, sizeof(buffer)) => strlen("hello"); + memcmp(buffer, "hello", strlen("hello")) => 0; + lfs_file_close(&lfs, &file[0]); + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Deferred file attributes ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_file_open(&lfs, &file[0], "hello/hello", LFS_O_RDWR) => 0; + + struct lfs_attr attr[] = { + {'B', "gggg", 4}, + {'C', "", 0}, + {'D', "hhhh", 4}, + }; + + lfs_file_setattrs(&lfs, &file[0], attr, 3) => 0; + lfs_file_getattrs(&lfs, &file[0], (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; + memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; + memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; + + lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "fffffffff", 9) => 0; + memcmp(buffer+9, "ccccc\0\0\0\0", 9) => 0; + memcmp(buffer+18, "\0\0\0\0\0\0\0\0\0", 9) => 0; + + lfs_file_sync(&lfs, &file[0]) => 0; + lfs_getattrs(&lfs, "hello/hello", (struct lfs_attr[]){ + {'B', buffer, 9}, + {'C', buffer+9, 9}, + {'D', buffer+18, 9}}, 3) => 0; + memcmp(buffer, "gggg\0\0\0\0\0", 9) => 0; + memcmp(buffer+9, "\0\0\0\0\0\0\0\0\0", 9) => 0; + memcmp(buffer+18, "hhhh\0\0\0\0\0", 9) => 0; + + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/tests/test_entries.sh b/tests/test_entries.sh new file mode 100755 index 00000000..447d9bca --- /dev/null +++ b/tests/test_entries.sh @@ -0,0 +1,220 @@ +#!/bin/bash +set -eu + +# Note: These tests are intended for 512 byte inline size at different +# inline sizes they should still pass, but won't be testing anything + +echo "=== Directory tests ===" +function read_file { +cat << TEST + + size = $2; + lfs_file_open(&lfs, &file[0], "$1", LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; +TEST +} + +function write_file { +cat << TEST + + size = $2; + lfs_file_open(&lfs, &file[0], "$1", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; +TEST +} + +echo "--- Entry grow test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 20) + $(write_file "hi1" 20) + $(write_file "hi2" 20) + $(write_file "hi3" 20) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 20) + $(read_file "hi1" 200) + $(read_file "hi2" 20) + $(read_file "hi3" 20) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry shrink test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 20) + $(write_file "hi1" 200) + $(write_file "hi2" 20) + $(write_file "hi3" 20) + + $(read_file "hi1" 200) + $(write_file "hi1" 20) + + $(read_file "hi0" 20) + $(read_file "hi1" 20) + $(read_file "hi2" 20) + $(read_file "hi3" 20) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry spill test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 200) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry push spill test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 20) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry push spill two test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 20) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + $(write_file "hi4" 200) + + $(read_file "hi1" 20) + $(write_file "hi1" 200) + + $(read_file "hi0" 200) + $(read_file "hi1" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + $(read_file "hi4" 200) + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Entry drop test ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + $(write_file "hi0" 200) + $(write_file "hi1" 200) + $(write_file "hi2" 200) + $(write_file "hi3" 200) + + lfs_remove(&lfs, "hi1") => 0; + lfs_stat(&lfs, "hi1", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + $(read_file "hi2" 200) + $(read_file "hi3" 200) + + lfs_remove(&lfs, "hi2") => 0; + lfs_stat(&lfs, "hi2", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + $(read_file "hi3" 200) + + lfs_remove(&lfs, "hi3") => 0; + lfs_stat(&lfs, "hi3", &info) => LFS_ERR_NOENT; + $(read_file "hi0" 200) + + lfs_remove(&lfs, "hi0") => 0; + lfs_stat(&lfs, "hi0", &info) => LFS_ERR_NOENT; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Create too big ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'm', 200); + buffer[200] = '\0'; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Resize too big ---" +tests/test.py << TEST + lfs_format(&lfs, &cfg) => 0; + + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'm', 200); + buffer[200] = '\0'; + + size = 40; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 40; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + memset(wbuffer, 'c', size); + lfs_file_write(&lfs, &file[0], wbuffer, size) => size; + lfs_file_close(&lfs, &file[0]) => 0; + + size = 400; + lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_RDONLY) => 0; + lfs_file_read(&lfs, &file[0], rbuffer, size) => size; + memcmp(rbuffer, wbuffer, size) => 0; + lfs_file_close(&lfs, &file[0]) => 0; + lfs_unmount(&lfs) => 0; +TEST + +echo "--- Results ---" +tests/stats.py diff --git a/tests/test_paths.sh b/tests/test_paths.sh index f277e451..33843296 100755 --- a/tests/test_paths.sh +++ b/tests/test_paths.sh @@ -123,5 +123,23 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Max path test ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + memset(buffer, 'w', LFS_NAME_MAX+1); + buffer[LFS_NAME_MAX+2] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => LFS_ERR_NAMETOOLONG; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NAMETOOLONG; + + memcpy(buffer, "coffee/", strlen("coffee/")); + memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX+1); + buffer[strlen("coffee/")+LFS_NAME_MAX+2] = '\0'; + lfs_mkdir(&lfs, (char*)buffer) => LFS_ERR_NAMETOOLONG; + lfs_file_open(&lfs, &file[0], (char*)buffer, + LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_NAMETOOLONG; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Results ---" tests/stats.py From 1f8c5099458752f894eb37af44d6c19efec28860 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 21 Jun 2018 11:35:57 -0500 Subject: [PATCH 30/31] Changed license to BSD-3-Clause For better compatibility with GPL v2 With permissions from: - aldot - Sim4n6 - jrast --- LICENSE.md | 183 +++++++--------------------------------------- emubd/lfs_emubd.c | 15 +--- emubd/lfs_emubd.h | 15 +--- lfs.c | 15 +--- lfs.h | 15 +--- lfs_util.c | 15 +--- lfs_util.h | 15 +--- 7 files changed, 39 insertions(+), 234 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index 59cd3f8a..f7949727 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,165 +1,36 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ +Copyright (c) 2017, Arm Limited. All rights reserved. -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: -1. Definitions. +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +- Neither the name of ARM nor the names of its contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. +--- -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. +*Note*: +Individual files contain the following tag instead of the full license text. -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. + SPDX-License-Identifier: BSD-3-Clause -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. +This enables machine processing of license information based on the SPDX +License Identifiers that are here available: http://spdx.org/licenses/ diff --git a/emubd/lfs_emubd.c b/emubd/lfs_emubd.c index b1595963..ed3414ab 100644 --- a/emubd/lfs_emubd.c +++ b/emubd/lfs_emubd.c @@ -1,19 +1,8 @@ /* * Block device emulated on standard files * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #include "emubd/lfs_emubd.h" diff --git a/emubd/lfs_emubd.h b/emubd/lfs_emubd.h index 4f87ccec..85718de1 100644 --- a/emubd/lfs_emubd.h +++ b/emubd/lfs_emubd.h @@ -1,19 +1,8 @@ /* * Block device emulated on standard files * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #ifndef LFS_EMUBD_H #define LFS_EMUBD_H diff --git a/lfs.c b/lfs.c index 49b4f3bc..c892fe99 100644 --- a/lfs.c +++ b/lfs.c @@ -1,19 +1,8 @@ /* * The little filesystem * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #include "lfs.h" #include "lfs_util.h" diff --git a/lfs.h b/lfs.h index 3fd3ba13..4e75ecb9 100644 --- a/lfs.h +++ b/lfs.h @@ -1,19 +1,8 @@ /* * The little filesystem * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #ifndef LFS_H #define LFS_H diff --git a/lfs_util.c b/lfs_util.c index 3a2a351b..9ca0756d 100644 --- a/lfs_util.c +++ b/lfs_util.c @@ -1,19 +1,8 @@ /* * lfs util functions * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #include "lfs_util.h" diff --git a/lfs_util.h b/lfs_util.h index 3527ce6c..51005dc3 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -1,19 +1,8 @@ /* * lfs utility functions * - * Copyright (c) 2017 ARM Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Copyright (c) 2017, Arm Limited. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause */ #ifndef LFS_UTIL_H #define LFS_UTIL_H From c23481cac22a8d6c37cd72de7c5c80336649490f Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 10 Jul 2018 08:44:00 -0500 Subject: [PATCH 31/31] Fixed script issue with bash expansion inside makefile parameter This was causing code sizes to be reported with several of the logging functions still built in. A useful number, but not the minimum achievable code size. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 54a5771e..92111061 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ script: # compile and find the code size with the smallest configuration - make clean size OBJ="$(ls lfs*.o | tr '\n' ' ')" - CFLAGS+="-DLFS_NO{ASSERT,DEBUG,WARN,ERROR}" + CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR" | tee sizes # update status if we succeeded, compare with master if possible