Reverted removal of 1-bit counter threaded through tags

Initially I thought the fcrc would be sufficient for all of the
end-of-commit context, since indicating that there is a new commit is a
simple as invalidating the fcrc. But it turns out there are cases that
make this impossible.

The surprising, and actually common, case, is that of an fcrc that
will end up containing a full commit. This is common as soon as the
prog_size is big, as small commits are padded to the prog_size at
minimum.

  .------------------. \
  |     metadata     | |
  |                  | |
  |                  | +-.
  |------------------| | |
  |   foward CRC ------------.
  |------------------| / |   |
  |   commit CRC    -----'   |
  |------------------|       |
  |     padding      |       |
  |                  |       |
  |------------------| \   \ |
  |     metadata     | |   | |
  |                  | +-. | |
  |                  | | | +-'
  |------------------| / | |
  |   commit CRC --------' |
  |------------------|     |
  |                  |     /
  '------------------'

When the commit + crc is all contained in the fcrc, something silly
happens with the math behind crcs. Everything in the commit gets
canceled out:

  crc(m) = m(x) x^|P|-1 mod P(x)

  m ++ crc(m) = m(x) x^|P|-1 + (m(x) x^|P|-1 mod P(x))

  crc(m ++ crc(m)) = (m(x) x^|P|-1 + (m(x) x^|P|-1 mod P(x))) x^|P|-1 mod P(x)

  crc(m ++ crc(m)) = (m(x) x^|P|-1 + m(x) x^|P|-1) x^|P|-1 mod P(x)

  crc(m ++ crc(m)) = 0 * x^|P|-1 mod P(x)

This is the reason the crc of a message + naive crc is zero. Even with an
initializer/bit-fiddling, the crc of the whole commit ends up as some
constant.

So no manipulation of the commit can change the fcrc...

But even if this did work, or we changed this scheme to use two
different checksums, it would still require calculating the fcrc of
the whole commit to know if we need to tweak the first bit to invalidate
the unlikely-but-problematic case where we happen to match the fcrc. This
would add a large amount of complexity to the commit code.

It's much simpler and cheaper to keep the 1-bit counter in the tag, even
if it adds another moving part to the system.

lfs.c

@@ -1078,6 +1078,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
         bool tempsplit = false;
         lfs_stag_t tempbesttag = besttag;
 
+        // assume not erased until proven otherwise
+        bool maybeerased = false;
         bool hasfcrc = false;
         struct lfs_fcrc fcrc;
 
@@ -1095,24 +1097,26 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
                     // can't continue?
-                    dir->erased = false;
                     break;
                 }
                 return err;
             }
 
             crc = lfs_crc(crc, &tag, sizeof(tag));
-            tag = (lfs_frombe32(tag) ^ ptag) & 0x7fffffff;
+            tag = lfs_frombe32(tag) ^ ptag;
 
+            // next commit not yet programmed?
+            if (!lfs_tag_isvalid(tag)) {
+                maybeerased = true;
+                break;
             // out of range?
-            if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
-                dir->erased = false;
+            } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
                 break;
             }
 
             ptag = tag;
 
-            if (lfs_tag_type2(tag) == LFS_TYPE_CRC) {
+            if (lfs_tag_type2(tag) == LFS_TYPE_CCRC) {
                 // check the crc attr
                 uint32_t dcrc;
                 err = lfs_bd_read(lfs,
@@ -1120,7 +1124,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                         dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc));
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
                         break;
                     }
                     return err;
@@ -1128,10 +1131,12 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 dcrc = lfs_fromle32(dcrc);
 
                 if (crc != dcrc) {
-                    dir->erased = false;
                     break;
                 }
 
+                // reset the next bit if we need to
+                ptag ^= (lfs_tag_t)(lfs_tag_chunk(tag) & 1U) << 31;
+
                 // toss our crc into the filesystem seed for
                 // pseudorandom numbers, note we use another crc here
                 // as a collection function because it is sufficiently
@@ -1147,50 +1152,14 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 dir->tail[1] = temptail[1];
                 dir->split = tempsplit;
 
-                // check for an fcrc matching the next prog's erased state, if
-                // this failed most likely a previous prog was interrupted, we
-                // need a new erase
-                if (hasfcrc) {
-                    // this may look inefficient, but since cache_size is
-                    // probably > prog_size, the data will always remain in
-                    // cache for the next iteration
-
-                    // first read the leading byte, this always contains a bit
-                    // we can perturb to avoid writes that don't change the fcrc
-                    uint8_t eperturb;
-                    err = lfs_bd_read(lfs,
-                            NULL, &lfs->rcache, lfs->cfg->block_size,
-                            dir->pair[0], dir->off, &eperturb, 1);
-                    if (err && err != LFS_ERR_CORRUPT) {
-                        return err;
-                    }
-
-                    // perturb valid bit?
-                    dir->etag |= (0x80 & ~eperturb) << 24;
-
-                    // crc the full prog_size, don't bother avoiding a reread
-                    // of the eperturb, it should still be in our cache
-                    uint32_t ecrc = 0xffffffff;
-                    err = lfs_bd_crc(lfs,
-                            NULL, &lfs->rcache, lfs->cfg->block_size,
-                            dir->pair[0], dir->off, fcrc.size, &ecrc);
-                    if (err && err != LFS_ERR_CORRUPT) {
-                        return err;
-                    }
-
-                    // found beginning of erased part?
-                    if (ecrc == fcrc.crc) {
-                        dir->erased = true;
-                        break;
-                    }
-                }
-
                 // reset crc
                 crc = 0xffffffff;
-                hasfcrc = false;
                 continue;
             }
 
+            // fcrc is only valid when last tag was a crc
+            hasfcrc = false;
+
             // crc the entry first, hopefully leaving it in the cache
             err = lfs_bd_crc(lfs,
                     NULL, &lfs->rcache, lfs->cfg->block_size,
@@ -1198,7 +1167,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                     lfs_tag_dsize(tag)-sizeof(tag), &crc);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
-                    dir->erased = false;
                     break;
                 }
                 return err;
@@ -1228,7 +1196,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                         dir->pair[0], off+sizeof(tag), &temptail, 8);
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
                         break;
                     }
                     return err;
@@ -1241,7 +1208,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                         &fcrc, sizeof(fcrc));
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
                         break;
                     }
                 }
@@ -1256,7 +1222,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                         dir->pair[0], off+sizeof(tag)});
                 if (res < 0) {
                     if (res == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
                         break;
                     }
                     return res;
@@ -1278,35 +1243,54 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             }
         }
 
-        // consider what we have good enough
-        if (dir->off > 0) {
-            // synthetic move
-            if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) {
-                if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) {
-                    besttag |= 0x80000000;
-                } else if (besttag != -1 &&
-                        lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) {
-                    besttag -= LFS_MKTAG(0, 1, 0);
-                }
-            }
+        // found no valid commits?
+        if (dir->off == 0) {
+            // try the other block?
+            lfs_pair_swap(dir->pair);
+            dir->rev = revs[(r+1)%2];
+            continue;
+        }
 
-            // found tag? or found best id?
-            if (id) {
-                *id = lfs_min(lfs_tag_id(besttag), dir->count);
+        // did we end on a valid commit? we may have an erased block
+        dir->erased = false;
+        if (maybeerased && hasfcrc && dir->off % lfs->cfg->prog_size == 0) {
+            // check for an fcrc matching the next prog's erased state, if
+            // this failed most likely a previous prog was interrupted, we
+            // need a new erase
+            uint32_t fcrc_ = 0xffffffff;
+            int err = lfs_bd_crc(lfs,
+                    NULL, &lfs->rcache, lfs->cfg->block_size,
+                    dir->pair[0], dir->off, fcrc.size, &fcrc_);
+            if (err && err != LFS_ERR_CORRUPT) {
+                return err;
             }
 
-            if (lfs_tag_isvalid(besttag)) {
-                return besttag;
-            } else if (lfs_tag_id(besttag) < dir->count) {
-                return LFS_ERR_NOENT;
-            } else {
-                return 0;
+            // found beginning of erased part?
+            dir->erased = (fcrc_ == fcrc.crc);
+        }
+
+        // synthetic move
+        if (lfs_gstate_hasmovehere(&lfs->gdisk, dir->pair)) {
+            if (lfs_tag_id(lfs->gdisk.tag) == lfs_tag_id(besttag)) {
+                besttag |= 0x80000000;
+            } else if (besttag != -1 &&
+                    lfs_tag_id(lfs->gdisk.tag) < lfs_tag_id(besttag)) {
+                besttag -= LFS_MKTAG(0, 1, 0);
             }
         }
 
-        // failed, try the other block?
-        lfs_pair_swap(dir->pair);
-        dir->rev = revs[(r+1)%2];
+        // found tag? or found best id?
+        if (id) {
+            *id = lfs_min(lfs_tag_id(besttag), dir->count);
+        }
+
+        if (lfs_tag_isvalid(besttag)) {
+            return besttag;
+        } else if (lfs_tag_id(besttag) < dir->count) {
+            return LFS_ERR_NOENT;
+        } else {
+            return 0;
+        }
     }
 
     LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}",
@@ -1598,8 +1582,9 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
     // padding is not crced, which lets fetches skip padding but
     // makes committing a bit more complicated
     while (commit->off < end) {
-        lfs_off_t noff = lfs_min(end - (commit->off+sizeof(lfs_tag_t)), 0x3fe)
-                + (commit->off+sizeof(lfs_tag_t));
+        lfs_off_t noff = (
+                lfs_min(end - (commit->off+sizeof(lfs_tag_t)), 0x3fe)
+                + (commit->off+sizeof(lfs_tag_t)));
         // too large for crc tag? need padding commits
         if (noff < end) {
             noff = lfs_min(noff, end - 5*sizeof(uint32_t));
@@ -1607,7 +1592,7 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
 
         // space for fcrc?
         uint8_t eperturb = -1;
-        if (noff < lfs->cfg->block_size) {
+        if (noff >= end && noff <= lfs->cfg->block_size - lfs->cfg->prog_size) {
             // first read the leading byte, this always contains a bit
             // we can perturb to avoid writes that don't change the fcrc
             int err = lfs_bd_read(lfs,
@@ -1619,15 +1604,9 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
 
             // find the expected fcrc, don't bother avoiding a reread
             // of the eperturb, it should still be in our cache
-            struct lfs_fcrc fcrc = {
-                // if our commit is a padding commit, we only care about
-                // invalidating outdated commits if there is a partial write, 
-                // so we fcrc the minimum amount (1 byte)
-                .size=(noff < end ? 1 : lfs->cfg->prog_size),
-                .crc=0xffffffff,
-            };
+            struct lfs_fcrc fcrc = {.size=lfs->cfg->prog_size, .crc=0xffffffff};
             err = lfs_bd_crc(lfs,
-                    NULL, &lfs->rcache, fcrc.size,
+                    NULL, &lfs->rcache, lfs->cfg->prog_size,
                     commit->block, noff, fcrc.size, &fcrc.crc);
             if (err && err != LFS_ERR_CORRUPT) {
                 return err;
@@ -1647,7 +1626,8 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
             lfs_tag_t tag;
             uint32_t crc;
         } ccrc;
-        lfs_tag_t ntag = LFS_MKTAG(LFS_TYPE_CCRC, 0x3ff,
+        lfs_tag_t ntag = LFS_MKTAG(
+                LFS_TYPE_CCRC + (((uint8_t)~eperturb) >> 7), 0x3ff,
                 noff - (commit->off+sizeof(lfs_tag_t)));
         ccrc.tag = lfs_tobe32(ntag ^ commit->ptag);
         commit->crc = lfs_crc(commit->crc, &ccrc.tag, sizeof(lfs_tag_t));
@@ -1668,15 +1648,19 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
 
         commit->off = noff;
         // perturb valid bit?
-        commit->ptag = ntag | ((0x80 & ~eperturb) << 24);
+        commit->ptag = ntag ^ ((0x80 & ~eperturb) << 24);
         // reset crc for next commit
         commit->crc = 0xffffffff;
-    }
 
-    // flush buffers
-    int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false);
-    if (err) {
-        return err;
+        // manually flush here since we don't prog the padding, this confuses
+        // the caching layer
+        if (noff >= end || noff >= lfs->pcache.off + lfs->cfg->cache_size) {
+            // flush buffers
+            int err = lfs_bd_sync(lfs, &lfs->pcache, &lfs->rcache, false);
+            if (err) {
+                return err;
+            }
+        }
     }
 
     // successful commit, check checksums to make sure
@@ -1685,7 +1669,7 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
     // case if they are corrupted we would have had to compact anyways
     lfs_off_t off = commit->begin;
     uint32_t crc = 0xffffffff;
-    err = lfs_bd_crc(lfs,
+    int err = lfs_bd_crc(lfs,
             NULL, &lfs->rcache, off1+sizeof(uint32_t),
             commit->block, off, off1-off, &crc);
     if (err) {

lfs.h

@@ -112,7 +112,7 @@ enum lfs_type {
     LFS_TYPE_SOFTTAIL       = 0x600,
     LFS_TYPE_HARDTAIL       = 0x601,
     LFS_TYPE_MOVESTATE      = 0x7ff,
-    LFS_TYPE_CCRC           = 0x502,
+    LFS_TYPE_CCRC           = 0x500,
     LFS_TYPE_FCRC           = 0x5ff,
 
     // internal chip sources

scripts/readmdir.py

@@ -23,7 +23,8 @@ TAG_TYPES = {
     'hardtail':     (0x7ff, 0x601),
     'gstate':       (0x700, 0x700),
     'movestate':    (0x7ff, 0x7ff),
-    'crc':          (0x780, 0x500),
+    'crc':          (0x700, 0x500),
+    'ccrc':         (0x780, 0x500),
     'fcrc':         (0x7ff, 0x5ff),
 }
 
@@ -121,12 +122,13 @@ class Tag:
         ntag = Tag(self.type, nid, self.size)
         if hasattr(self, 'off'):    ntag.off    = self.off
         if hasattr(self, 'data'):   ntag.data   = self.data
-        if hasattr(self, 'crc'):    ntag.crc    = self.crc
+        if hasattr(self, 'ccrc'):   ntag.crc    = self.crc
         if hasattr(self, 'erased'): ntag.erased = self.erased
         return ntag
 
     def typerepr(self):
-        if (self.is_('crc') and getattr(self, 'crc', 0xffffffff) != 0xffffffff):
+        if (self.is_('ccrc')
+                and getattr(self, 'ccrc', 0xffffffff) != 0xffffffff):
             crc_status = ' (bad)'
         elif self.is_('fcrc') and getattr(self, 'erased', False):
             crc_status = ' (era)'
@@ -187,8 +189,8 @@ class MetadataPair:
 
         self.rev, = struct.unpack('<I', block[0:4])
         crc = binascii.crc32(block[0:4])
-        etag = None
-        estate = None
+        fcrctag = None
+        fcrcdata = None
 
         # parse tags
         corrupt = False
@@ -202,7 +204,7 @@ class MetadataPair:
             tag = Tag((int(tag) ^ ntag) & 0x7fffffff)
             tag.off = off + 4
             tag.data = block[off+4:off+tag.dsize]
-            if tag.is_('crc'):
+            if tag.is_('ccrc'):
                 crc = binascii.crc32(block[off:off+2*4], crc)
             else:
                 crc = binascii.crc32(block[off:off+tag.dsize], crc)
@@ -212,26 +214,28 @@ class MetadataPair:
             self.all_.append(tag)
 
             if tag.is_('fcrc') and len(tag.data) == 8:
-                etag = tag
-                estate = struct.unpack('<II', tag.data)
-            elif tag.is_('crc'):
+                fcrctag = tag
+                fcrcdata = struct.unpack('<II', tag.data)
+            elif tag.is_('ccrc'):
                 # is valid commit?
                 if crc != 0xffffffff:
                     corrupt = True
                 if not corrupt:
                     self.log = self.all_.copy()
                     # end of commit?
-                    if estate:
-                        esize, ecrc = estate
-                        dcrc = 0xffffffff ^ binascii.crc32(block[off:off+esize])
-                        if ecrc == dcrc:
-                            etag.erased = True
+                    if fcrcdata:
+                        fcrcsize, fcrc = fcrcdata
+                        fcrc_ = 0xffffffff ^ binascii.crc32(
+                            block[off:off+fcrcsize])
+                        if fcrc_ == fcrc:
+                            fcrctag.erased = True
                             corrupt = True
 
                 # reset tag parsing
                 crc = 0
-                etag = None
-                estate = None
+                tag = Tag(int(tag) ^ ((tag.type & 1) << 31))
+                fcrctag = None
+                fcrcdata = None
 
         # find active ids
         self.ids = list(it.takewhile(
@@ -241,7 +245,7 @@ class MetadataPair:
         # find most recent tags
         self.tags = []
         for tag in self.log:
-            if tag.is_('crc') or tag.is_('fcrc') or tag.is_('splice'):
+            if tag.is_('crc') or tag.is_('splice'):
                 continue
             elif tag.id == 0x3ff:
                 if tag in self and self[tag] is tag: