Modified valid bit to provide an early check on all tags

The valid bit present in tags is a requirement to properly detect the
end of commits in metadata logs. The way it works is that the CRC entry is
allowed to specify what is needed from the next tag's valid bit. If it's
incorrect, we've reached the end of the commit. We then set the valid bit to
indicate when we tried to program a new commit. If we lose power, this
commit will still be thrown out by a bad checksum.

However, the valid bit is unused outside of the CRC entry. Here we turn on the
valid bit for all tags, which means we have a decent chance of exiting early
if we hit a half-written commit. We still need to guarantee detection of
the valid bit on commits following the CRC entry, so we allow the CRC
entry to flip the expected valid bit.

The only tricky part is what valid bit we expect by default, since this
is used on the first commit on a metadata log. Here we default to a 1,
which gives us the fastest exit on blocks that erase to 0. This is
because blocks that erase to 1s will implicitly flip the valid bit of
the next tag, allowing us to exit on the next tag.

If we defaulted to 0, we could exit faster on disks that erase to 1, but
would need to scan the entire block on disks that erase to 0 before we
realize a CRC commit is never coming.

lfs.c

@@ -385,7 +385,7 @@ static inline uint16_t lfs_tag_type(uint32_t tag) {
 }
 
 static inline uint16_t lfs_tag_subtype(uint32_t tag) {
-    return (tag & 0x7c000000) >> 22;
+    return ((tag & 0x7c000000) >> 26) << 4;
 }
 
 static inline uint16_t lfs_tag_id(uint32_t tag) {
@@ -470,7 +470,7 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off,
     while (off >= 2*sizeof(tag)+lfs_tag_size(tag)) {
         off -= sizeof(tag)+lfs_tag_size(tag);
 
-        if (lfs_tag_type(tag) == LFS_TYPE_CRC && stopatcommit) {
+        if (lfs_tag_subtype(tag) == LFS_TYPE_CRC && stopatcommit) {
             break;
         } else if (lfs_tag_type(tag) == LFS_TYPE_DELETE) {
             if (lfs_tag_id(tag) <= lfs_tag_id(gettag + getdiff)) {
@@ -502,6 +502,7 @@ static int32_t lfs_commit_get(lfs_t *lfs, lfs_block_t block, lfs_off_t off,
             return err;
         }
         tag ^= lfs_fromle32(ntag);
+        tag &= 0x7fffffff;
     }
 
     return LFS_ERR_NOENT;
@@ -632,8 +633,7 @@ static int lfs_commit_move(lfs_t *lfs, struct lfs_commit *commit,
                 return err;
             }
 
-            ntag = lfs_fromle32(ntag);
-            ntag ^= tag;
+            ntag = lfs_fromle32(ntag) ^ tag;
             tag |= 0x80000000;
         }
 
@@ -687,7 +687,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) {
             lfs->cfg->prog_size);
 
     // read erased state from next program unit
-    uint32_t tag = 0;
+    uint32_t tag;
     int err = lfs_bd_read(lfs,
             &lfs->pcache, &lfs->rcache, sizeof(tag),
             commit->block, off, &tag, sizeof(tag));
@@ -696,10 +696,9 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) {
     }
 
     // build crc tag
-    tag = lfs_fromle32(tag);
-    tag = (0x80000000 & ~tag) |
-            LFS_MKTAG(LFS_TYPE_CRC, 0x3ff,
-                off - (commit->off+sizeof(uint32_t)));
+    bool reset = ~lfs_fromle32(tag) >> 31;
+    tag = LFS_MKTAG(LFS_TYPE_CRC + reset,
+            0x3ff, off - (commit->off+sizeof(uint32_t)));
 
     // write out crc
     uint32_t footer[2];
@@ -713,7 +712,7 @@ static int lfs_commit_crc(lfs_t *lfs, struct lfs_commit *commit) {
         return err;
     }
     commit->off += sizeof(tag)+lfs_tag_size(tag);
-    commit->ptag = tag;
+    commit->ptag = tag ^ (reset << 31);
 
     // flush buffers
     err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false);
@@ -774,7 +773,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) {
 
     // set defaults
     dir->off = sizeof(dir->rev);
-    dir->etag = 0;
+    dir->etag = 0xffffffff;
     dir->count = 0;
     dir->tail[0] = 0xffffffff;
     dir->tail[1] = 0xffffffff;
@@ -817,7 +816,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs,
     // load blocks and check crc
     for (int i = 0; i < 2; i++) {
         lfs_off_t off = sizeof(dir->rev);
-        uint32_t ptag = 0;
+        uint32_t ptag = 0xffffffff;
         uint32_t crc = 0xffffffff;
 
         dir->rev = lfs_tole32(rev[0]);
@@ -851,8 +850,8 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs,
             tag = lfs_fromle32(tag) ^ ptag;
 
             // next commit not yet programmed
-            if (lfs_tag_type(ptag) == LFS_TYPE_CRC && !lfs_tag_isvalid(tag)) {
-                dir->erased = true;
+            if (!lfs_tag_isvalid(tag)) {
+                dir->erased = (lfs_tag_subtype(ptag) == LFS_TYPE_CRC);
                 break;
             }
 
@@ -862,7 +861,7 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs,
                 break;
             }
 
-            if (lfs_tag_type(tag) == LFS_TYPE_CRC) {
+            if (lfs_tag_subtype(tag) == LFS_TYPE_CRC) {
                 // check the crc attr
                 uint32_t dcrc;
                 err = lfs_bd_read(lfs,
@@ -882,6 +881,10 @@ static int32_t lfs_dir_fetchmatch(lfs_t *lfs,
                     break;
                 }
 
+                // reset the next bit if we need to
+                tag ^= (lfs_tag_type(tag) & 1) << 31;
+
+                // update with what's found so far
                 foundtag = tempfoundtag;
                 dir->off = off + sizeof(tag)+lfs_tag_size(tag);
                 dir->etag = tag;
@@ -1096,7 +1099,7 @@ commit:
         // setup commit state
         commit.off = 0;
         commit.crc = 0xffffffff;
-        commit.ptag = 0;
+        commit.ptag = 0xffffffff;
 
         // space is complicated, we need room for tail, crc, globals,
         // cleanup delete, and we cap at half a block to give room

tests/corrupt.py

@@ -11,7 +11,7 @@ def corrupt(block):
         file.read(4)
 
         # go to last commit
-        tag = 0
+        tag = 0xffffffff
         while True:
             try:
                 ntag, = struct.unpack('<I', file.read(4))

tests/debug.py

@@ -12,7 +12,7 @@ TYPES = {
     (0x1f0, 0x080): 'globals',
     (0x1ff, 0x0c0): 'tail soft',
     (0x1ff, 0x0c1): 'tail hard',
-    (0x1ff, 0x0f0): 'crc',
+    (0x1f0, 0x0f0): 'crc',
     (0x1ff, 0x040): 'struct dir',
     (0x1ff, 0x041): 'struct inline',
     (0x1ff, 0x042): 'struct ctz',
@@ -63,7 +63,7 @@ def main(*blocks):
     print "%-4s  %-8s  %-14s  %3s  %3s  %s" % (
         'off', 'tag', 'type', 'id', 'len', 'dump')
 
-    tag = 0
+    tag = 0xffffffff
     off = 4
     while True:
         try:
@@ -79,23 +79,25 @@ def main(*blocks):
         type = (tag & 0x7fc00000) >> 22
         id   = (tag & 0x003ff000) >> 12
         size = (tag & 0x00000fff) >> 0
+        iscrc = (type & 0x1f0) == 0x0f0
 
         data = file.read(size)
-        if type == 0x0f0:
+        if iscrc:
             crc = binascii.crc32(data[:4], crc)
         else:
             crc = binascii.crc32(data, crc)
 
         print '%04x: %08x  %-14s  %3s  %3d  %-23s  %-8s' % (
             off, tag,
-            typeof(type) + (' bad!' if type == 0x0f0 and ~crc else ''),
+            typeof(type) + (' bad!' if iscrc and ~crc else ''),
             id if id != 0x3ff else '.', size,
             ' '.join('%02x' % ord(c) for c in data[:8]),
             ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
 
         off += tag & 0xfff
-        if type == 0x0f0:
+        if iscrc:
             crc = 0
+            tag ^= (type & 1) << 31
 
     return 0