Added support for writing on-disk version lfs2.0

The intention is to help interop with older minor versions of littlefs.

Unfortunately, since lfs2.0 drivers cannot mount lfs2.1 images, there are
situations where it would be useful to write to write strictly lfs2.0
compatible images. The solution here adds a "disk_version" configuration
option which determines the behavior of lfs2.1 dependent features.

Normally you would expect this to only change write behavior. But since the
main change in lfs2.1 increased validation of erased data, we also need to
skip this extra validation (fcrc) or see terrible slowdowns when writing.

lfs.c

@@ -518,6 +518,24 @@ static void lfs_mlist_append(lfs_t *lfs, struct lfs_mlist *mlist) {
     lfs->mlist = mlist;
 }
 
+// some other filesystem operations
+static uint32_t lfs_fs_disk_version(lfs_t *lfs) {
+    if (lfs->cfg->disk_version) {
+        return lfs->cfg->disk_version;
+    } else {
+        return LFS_DISK_VERSION;
+    }
+}
+
+static uint16_t lfs_fs_disk_version_major(lfs_t *lfs) {
+    return 0xffff & (lfs_fs_disk_version(lfs) >> 16);
+
+}
+
+static uint16_t lfs_fs_disk_version_minor(lfs_t *lfs) {
+    return 0xffff & (lfs_fs_disk_version(lfs) >> 0);
+}
+
 
 /// Internal operations predeclared here ///
 #ifndef LFS_READONLY
@@ -1111,7 +1129,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
 
             // next commit not yet programmed?
             if (!lfs_tag_isvalid(tag)) {
-                maybeerased = true;
+                // we only might be erased if the last tag was a crc
+                maybeerased = (lfs_tag_type2(ptag) == LFS_TYPE_CCRC);
                 break;
             // out of range?
             } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
@@ -1156,14 +1175,11 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 dir->tail[1] = temptail[1];
                 dir->split = tempsplit;
 
-                // reset crc
+                // reset crc, hasfcrc
                 crc = 0xffffffff;
                 continue;
             }
 
-            // fcrc is only valid when last tag was a crc
-            hasfcrc = false;
-
             // crc the entry first, hopefully leaving it in the cache
             err = lfs_bd_crc(lfs,
                     NULL, &lfs->rcache, lfs->cfg->block_size,
@@ -1257,20 +1273,30 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
 
         // did we end on a valid commit? we may have an erased block
         dir->erased = false;
-        if (maybeerased && hasfcrc && dir->off % lfs->cfg->prog_size == 0) {
-            // check for an fcrc matching the next prog's erased state, if
-            // this failed most likely a previous prog was interrupted, we
-            // need a new erase
-            uint32_t fcrc_ = 0xffffffff;
-            int err = lfs_bd_crc(lfs,
-                    NULL, &lfs->rcache, lfs->cfg->block_size,
-                    dir->pair[0], dir->off, fcrc.size, &fcrc_);
-            if (err && err != LFS_ERR_CORRUPT) {
-                return err;
-            }
+        if (maybeerased && dir->off % lfs->cfg->prog_size == 0) {
+            // note versions < lfs2.1 did not have fcrc tags, if
+            // we're < lfs2.1 treat missing fcrc as erased data
+            //
+            // we don't strictly need to do this, but otherwise writing
+            // to lfs2.0 disks becomes very inefficient
+            if (lfs_fs_disk_version(lfs) < 0x00020001) {
+                dir->erased = true;
+
+            } else if (hasfcrc) {
+                // check for an fcrc matching the next prog's erased state, if
+                // this failed most likely a previous prog was interrupted, we
+                // need a new erase
+                uint32_t fcrc_ = 0xffffffff;
+                int err = lfs_bd_crc(lfs,
+                        NULL, &lfs->rcache, lfs->cfg->block_size,
+                        dir->pair[0], dir->off, fcrc.size, &fcrc_);
+                if (err && err != LFS_ERR_CORRUPT) {
+                    return err;
+                }
 
-            // found beginning of erased part?
-            dir->erased = (fcrc_ == fcrc.crc);
+                // found beginning of erased part?
+                dir->erased = (fcrc_ == fcrc.crc);
+            }
         }
 
         // synthetic move
@@ -1606,22 +1632,29 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
                 return err;
             }
 
-            // find the expected fcrc, don't bother avoiding a reread
-            // of the eperturb, it should still be in our cache
-            struct lfs_fcrc fcrc = {.size=lfs->cfg->prog_size, .crc=0xffffffff};
-            err = lfs_bd_crc(lfs,
-                    NULL, &lfs->rcache, lfs->cfg->prog_size,
-                    commit->block, noff, fcrc.size, &fcrc.crc);
-            if (err && err != LFS_ERR_CORRUPT) {
-                return err;
-            }
+            // unfortunately fcrcs break mdir fetching < lfs2.1, so only write
+            // these if we're a >= lfs2.1 filesystem
+            if (lfs_fs_disk_version(lfs) >= 0x00020001) {
+                // find the expected fcrc, don't bother avoiding a reread
+                // of the eperturb, it should still be in our cache
+                struct lfs_fcrc fcrc = {
+                    .size = lfs->cfg->prog_size,
+                    .crc = 0xffffffff
+                };
+                err = lfs_bd_crc(lfs,
+                        NULL, &lfs->rcache, lfs->cfg->prog_size,
+                        commit->block, noff, fcrc.size, &fcrc.crc);
+                if (err && err != LFS_ERR_CORRUPT) {
+                    return err;
+                }
 
-            lfs_fcrc_tole32(&fcrc);
-            err = lfs_dir_commitattr(lfs, commit,
-                    LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)),
-                    &fcrc);
-            if (err) {
-                return err;
+                lfs_fcrc_tole32(&fcrc);
+                err = lfs_dir_commitattr(lfs, commit,
+                        LFS_MKTAG(LFS_TYPE_FCRC, 0x3ff, sizeof(struct lfs_fcrc)),
+                        &fcrc);
+                if (err) {
+                    return err;
+                }
             }
         }
 
@@ -4052,6 +4085,13 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->cfg = cfg;
     int err = 0;
 
+    // this driver only supports minor version < current minor version
+    LFS_ASSERT(!lfs->cfg->disk_version || (
+            (0xffff & (lfs->cfg->disk_version >> 16))
+                    == LFS_DISK_VERSION_MAJOR
+                && (0xffff & (lfs->cfg->disk_version >> 0))
+                    <= LFS_DISK_VERSION_MINOR));
+
     // check that bool is a truthy-preserving type
     //
     // note the most common reason for this failure is a before-c99 compiler,
@@ -4209,7 +4249,7 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) {
 
         // write one superblock
         lfs_superblock_t superblock = {
-            .version     = LFS_DISK_VERSION,
+            .version     = lfs_fs_disk_version(lfs),
             .block_size  = lfs->cfg->block_size,
             .block_count = lfs->cfg->block_count,
             .name_max    = lfs->name_max,
@@ -4307,12 +4347,14 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
             // check version
             uint16_t major_version = (0xffff & (superblock.version >> 16));
             uint16_t minor_version = (0xffff & (superblock.version >>  0));
-            if ((major_version != LFS_DISK_VERSION_MAJOR ||
-                 minor_version > LFS_DISK_VERSION_MINOR)) {
+            if (major_version != lfs_fs_disk_version_major(lfs)
+                    || minor_version > lfs_fs_disk_version_minor(lfs)) {
                 LFS_ERROR("Invalid version "
                         "v%"PRIu16".%"PRIu16" != v%"PRIu16".%"PRIu16,
-                        major_version, minor_version,
-                        LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
+                        major_version,
+                        minor_version,
+                        lfs_fs_disk_version_major(lfs),
+                        lfs_fs_disk_version_minor(lfs));
                 err = LFS_ERR_INVAL;
                 goto cleanup;
             }
@@ -4320,11 +4362,13 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
             // found older minor version? set an in-device only bit in the
             // gstate so we know we need to rewrite the superblock before
             // the first write
-            if (minor_version < LFS_DISK_VERSION_MINOR) {
+            if (minor_version < lfs_fs_disk_version_minor(lfs)) {
                 LFS_DEBUG("Found older minor version "
                         "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16,
-                        major_version, minor_version,
-                        LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR);
+                        major_version,
+                        minor_version,
+                        lfs_fs_disk_version_major(lfs),
+                        lfs_fs_disk_version_minor(lfs));
                 // note this bit is reserved on disk, so fetching more gstate
                 // will not interfere here
                 lfs_fs_prepsuperblock(lfs, true);
@@ -4424,7 +4468,7 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) {
     // if the superblock is up-to-date, we must be on the most recent
     // minor version of littlefs
     if (!lfs_gstate_needssuperblock(&lfs->gstate)) {
-        fsinfo->disk_version = LFS_DISK_VERSION;
+        fsinfo->disk_version = lfs_fs_disk_version(lfs);
 
     // otherwise we need to read the minor version on disk
     } else {
@@ -4711,7 +4755,7 @@ static int lfs_fs_desuperblock(lfs_t *lfs) {
 
     // write a new superblock
     lfs_superblock_t superblock = {
-        .version     = LFS_DISK_VERSION,
+        .version     = lfs_fs_disk_version(lfs),
         .block_size  = lfs->cfg->block_size,
         .block_count = lfs->cfg->block_count,
         .name_max    = lfs->name_max,

lfs.h

@@ -263,6 +263,12 @@ struct lfs_config {
     // can help bound the metadata compaction time. Must be <= block_size.
     // Defaults to block_size when zero.
     lfs_size_t metadata_max;
+
+    // On-disk version to use when writing in the form of 16-bit major version
+    // + 16-bit minor version. This limiting metadata to what is supported by
+    // older minor versions. Note that some features will be lost. Defaults to 
+    // to the most recent minor version when zero.
+    uint32_t disk_version;
 };
 
 // File info structure

runners/test_runner.c

@@ -1346,6 +1346,7 @@ static void run_powerloss_none(
         .block_cycles       = BLOCK_CYCLES,
         .cache_size         = CACHE_SIZE,
         .lookahead_size     = LOOKAHEAD_SIZE,
+        .disk_version       = DISK_VERSION,
     };
 
     struct lfs_emubd_config bdcfg = {
@@ -1415,6 +1416,7 @@ static void run_powerloss_linear(
         .block_cycles       = BLOCK_CYCLES,
         .cache_size         = CACHE_SIZE,
         .lookahead_size     = LOOKAHEAD_SIZE,
+        .disk_version       = DISK_VERSION,
     };
 
     struct lfs_emubd_config bdcfg = {
@@ -1501,6 +1503,7 @@ static void run_powerloss_log(
         .block_cycles       = BLOCK_CYCLES,
         .cache_size         = CACHE_SIZE,
         .lookahead_size     = LOOKAHEAD_SIZE,
+        .disk_version       = DISK_VERSION,
     };
 
     struct lfs_emubd_config bdcfg = {
@@ -1585,6 +1588,7 @@ static void run_powerloss_cycles(
         .block_cycles       = BLOCK_CYCLES,
         .cache_size         = CACHE_SIZE,
         .lookahead_size     = LOOKAHEAD_SIZE,
+        .disk_version       = DISK_VERSION,
     };
 
     struct lfs_emubd_config bdcfg = {
@@ -1767,6 +1771,7 @@ static void run_powerloss_exhaustive(
         .block_cycles       = BLOCK_CYCLES,
         .cache_size         = CACHE_SIZE,
         .lookahead_size     = LOOKAHEAD_SIZE,
+        .disk_version       = DISK_VERSION,
     };
 
     struct lfs_emubd_config bdcfg = {

runners/test_runner.h

@@ -91,6 +91,7 @@ intmax_t test_define(size_t define);
 #define ERASE_CYCLES_i       8
 #define BADBLOCK_BEHAVIOR_i  9
 #define POWERLOSS_BEHAVIOR_i 10
+#define DISK_VERSION_i       11
 
 #define READ_SIZE           TEST_DEFINE(READ_SIZE_i)
 #define PROG_SIZE           TEST_DEFINE(PROG_SIZE_i)
@@ -103,6 +104,7 @@ intmax_t test_define(size_t define);
 #define ERASE_CYCLES        TEST_DEFINE(ERASE_CYCLES_i)
 #define BADBLOCK_BEHAVIOR   TEST_DEFINE(BADBLOCK_BEHAVIOR_i)
 #define POWERLOSS_BEHAVIOR  TEST_DEFINE(POWERLOSS_BEHAVIOR_i)
+#define DISK_VERSION        TEST_DEFINE(DISK_VERSION_i)
 
 #define TEST_IMPLICIT_DEFINES \
     TEST_DEF(READ_SIZE,          PROG_SIZE) \
@@ -115,9 +117,10 @@ intmax_t test_define(size_t define);
     TEST_DEF(ERASE_VALUE,        0xff) \
     TEST_DEF(ERASE_CYCLES,       0) \
     TEST_DEF(BADBLOCK_BEHAVIOR,  LFS_EMUBD_BADBLOCK_PROGERROR) \
-    TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP)
+    TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) \
+    TEST_DEF(DISK_VERSION,       0)
 
-#define TEST_IMPLICIT_DEFINE_COUNT 11
+#define TEST_IMPLICIT_DEFINE_COUNT 12
 #define TEST_GEOMETRY_DEFINE_COUNT 4
 
 

tests/test_compat.toml

@@ -60,7 +60,10 @@ code = '''
 
 # test we can mount in a new version
 [cases.test_compat_forward_mount]
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -88,7 +91,10 @@ code = '''
 # test we can read dirs in a new version
 [cases.test_compat_forward_read_dirs]
 defines.COUNT = 5
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -145,7 +151,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 4
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -232,7 +241,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 4
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -344,7 +356,10 @@ code = '''
 # test we can write dirs in a new version
 [cases.test_compat_forward_write_dirs]
 defines.COUNT = 10
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -408,7 +423,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 2
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -527,7 +545,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 2
-if = 'LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR'
+if = '''
+    LFS_DISK_VERSION_MAJOR == LFSP_DISK_VERSION_MAJOR
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     struct lfsp_config cfgp;
@@ -674,7 +695,10 @@ code = '''
 
 # test we can mount in an old version
 [cases.test_compat_backward_mount]
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the new version
     lfs_t lfs;
@@ -696,7 +720,10 @@ code = '''
 # test we can read dirs in an old version
 [cases.test_compat_backward_read_dirs]
 defines.COUNT = 5
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the new version
     lfs_t lfs;
@@ -748,7 +775,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 4
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the new version
     lfs_t lfs;
@@ -830,7 +860,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 4
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the new version
     lfs_t lfs;
@@ -937,7 +970,10 @@ code = '''
 # test we can write dirs in an old version
 [cases.test_compat_backward_write_dirs]
 defines.COUNT = 10
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the new version
     lfs_t lfs;
@@ -996,7 +1032,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 2
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     lfs_t lfs;
@@ -1110,7 +1149,10 @@ code = '''
 defines.COUNT = 5
 defines.SIZE = [4, 32, 512, 8192]
 defines.CHUNK = 2
-if = 'LFS_DISK_VERSION == LFSP_DISK_VERSION'
+if = '''
+    LFS_DISK_VERSION == LFSP_DISK_VERSION
+        && DISK_VERSION == 0
+'''
 code = '''
     // create the previous version
     lfs_t lfs;
@@ -1319,7 +1361,10 @@ code = '''
 # test that we correctly bump the minor version
 [cases.test_compat_minor_bump]
 in = 'lfs.c'
-if = 'LFS_DISK_VERSION_MINOR > 0'
+if = '''
+    LFS_DISK_VERSION_MINOR > 0
+        && DISK_VERSION == 0
+'''
 code = '''
     // create a superblock
     lfs_t lfs;

tests/test_powerloss.toml

@@ -90,7 +90,10 @@ code = '''
 
 # partial prog, may not be byte in order!
 [cases.test_powerloss_partial_prog]
-if = "PROG_SIZE < BLOCK_SIZE"
+if = '''
+    PROG_SIZE < BLOCK_SIZE
+        && (DISK_VERSION == 0 || DISK_VERSION >= 0x00020001)
+'''
 defines.BYTE_OFF = ["0", "PROG_SIZE-1", "PROG_SIZE/2"]
 defines.BYTE_VALUE = [0x33, 0xcc]
 in = "lfs.c"

tests/test_superblocks.toml

@@ -36,6 +36,7 @@ code = '''
 
 # test we can read superblock info through lfs_fs_stat
 [cases.test_superblocks_stat]
+if = 'DISK_VERSION == 0'
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -54,6 +55,7 @@ code = '''
 '''
 
 [cases.test_superblocks_stat_tweaked]
+if = 'DISK_VERSION == 0'
 defines.TWEAKED_NAME_MAX = 63
 defines.TWEAKED_FILE_MAX = '(1 << 16)-1'
 defines.TWEAKED_ATTR_MAX = 512