Cleaned up tag encoding, now with clear chunk field

Before, the tag format's type field was limited to 9-bits. This sounds
like a lot, but this field needed to encode up to 256 user-specified
types. This limited the flexibility of the encoded types. As time went
on, more bits in the type field were repurposed for various things,
leaving a rather fragile type field.

Here we make the jump to full 11-bit type fields. This comes at the cost
of a smaller length field, however the use of the length field was
always going to come with a RAM limitation. Rather than putting pressure
on RAM for inline files, the new type field lets us encode a chunk
number, splitting up inline files into multiple updatable units. This
actually pushes the theoretical inline max from 8KiB to 256KiB! (Note
that we only allow a single 1KiB chunk for now, chunky inline files
is just a theoretical future improvement).

Here is the new 32-bit tag format, note that there are multiple levels
of types which break down into more info:

[----            32             ----]
[1|--  11   --|--  10  --|--  10  --]
 ^.     ^     .     ^          ^- entry length
 |.     |     .     \------------ file id chunk info
 |.     \-----.------------------ type info (type3)
 \.-----------.------------------ valid bit
  [-3-|-- 8 --]
    ^     ^- chunk info
    \------- type info (type1)

Additionally, I've split the CREATE tag into separate SPLICE and NAME
tags. This simplified the new compact logic a bit. For now, littlefs
still follows the rule that a NAME tag precedes any other tags related
to a file, but this can change in the future.

lfs.h

@@ -45,25 +45,25 @@ typedef int32_t  lfs_soff_t;
 typedef uint32_t lfs_block_t;
 
 // Maximum name size in bytes, may be redefined to reduce the size of the
-// info struct. Limited to <= 8190. Stored in superblock and must be
+// info struct. Limited to <= 1022. Stored in superblock and must be
 // respected by other littlefs drivers.
 #ifndef LFS_NAME_MAX
-#define LFS_NAME_MAX 0xff
+#define LFS_NAME_MAX 255
 #endif
 
 // Maximum inline file size in bytes, may be redefined to limit RAM usage,
 // but littlefs will automatically limit the LFS_INLINE_MAX to the
-// configured cache_size. Limited to <= 8190. Stored in superblock and must
+// configured cache_size. Limited to <= 1022. Stored in superblock and must
 // be respected by other littlefs drivers.
 #ifndef LFS_INLINE_MAX
-#define LFS_INLINE_MAX 0x1ffe
+#define LFS_INLINE_MAX 1022
 #endif
 
 // Maximum size of custom attributes in bytes, may be redefined, but there is
-// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 8190. Stored
+// no real benefit to using a smaller LFS_ATTR_MAX. Limited to <= 1022. Stored
 // in superblock and must be respected by other littlefs drivers.
 #ifndef LFS_ATTR_MAX
-#define LFS_ATTR_MAX 0x1ffe
+#define LFS_ATTR_MAX 1022
 #endif
 
 // Maximum size of a file in bytes, may be redefined to limit to support other
@@ -98,31 +98,33 @@ enum lfs_error {
 // File types
 enum lfs_type {
     // file types
-    LFS_TYPE_REG            = 0x011,
-    LFS_TYPE_DIR            = 0x010,
+    LFS_TYPE_REG            = 0x001,
+    LFS_TYPE_DIR            = 0x002,
 
     // internally used types
-    LFS_TYPE_USERATTR       = 0x100,
-    LFS_TYPE_CREATE         = 0x000,
-    LFS_TYPE_DELETE         = 0x020,
-    LFS_TYPE_STRUCT         = 0x040,
-    LFS_TYPE_TAIL           = 0x080,
-    LFS_TYPE_SOFTTAIL       = 0x080,
-    LFS_TYPE_HARDTAIL       = 0x081,
-    LFS_TYPE_CRC            = 0x0a0,
-    LFS_TYPE_SUPERBLOCK     = 0x001,
-    LFS_TYPE_GLOBALS        = 0x0e0,
-
-    LFS_TYPE_DIRSTRUCT      = 0x040,
-    LFS_TYPE_INLINESTRUCT   = 0x041,
-    LFS_TYPE_CTZSTRUCT      = 0x042,
+    LFS_TYPE_SPLICE         = 0x400,
+    LFS_TYPE_NAME           = 0x000,
+    LFS_TYPE_STRUCT         = 0x200,
+    LFS_TYPE_USERATTR       = 0x300,
+    LFS_TYPE_FROM           = 0x100,
+    LFS_TYPE_TAIL           = 0x600,
+    LFS_TYPE_GLOBALS        = 0x700,
+    LFS_TYPE_CRC            = 0x500,
+
+    // internally used type specializations
+    LFS_TYPE_CREATE         = 0x401,
+    LFS_TYPE_DELETE         = 0x4ff,
+    LFS_TYPE_SUPERBLOCK     = 0x0ff,
+    LFS_TYPE_DIRSTRUCT      = 0x200,
+    LFS_TYPE_CTZSTRUCT      = 0x202,
+    LFS_TYPE_INLINESTRUCT   = 0x201,
+    LFS_TYPE_SOFTTAIL       = 0x600,
+    LFS_TYPE_HARDTAIL       = 0x601,
+    LFS_TYPE_MOVESTATE      = 0x7ff,
 
     // internal chip sources
-    LFS_TYPE_FROM           = 0x060,
-    LFS_FROM_MEM            = 0x000,
-    LFS_FROM_DISK           = 0x200,
-    LFS_FROM_MOVE           = 0x061,
-    LFS_FROM_USERATTRS      = 0x062,
+    LFS_FROM_MOVE           = 0x101,
+    LFS_FROM_USERATTRS      = 0x102,
 };
 
 // File open flags
@@ -314,8 +316,8 @@ typedef struct lfs_cache {
 typedef struct lfs_mdir {
     lfs_block_t pair[2];
     uint32_t rev;
-    uint32_t etag;
     lfs_off_t off;
+    uint32_t etag;
     uint16_t count;
     bool erased;
     bool split;

tests/corrupt.py

@@ -19,7 +19,7 @@ def corrupt(block):
                 break
 
             tag ^= ntag
-            size = (tag & 0x1fff) if (tag & 0x1fff) != 0x1fff else 0
+            size = (tag & 0x3ff) if (tag & 0x3ff) != 0x3ff else 0
             file.seek(size, os.SEEK_CUR)
 
         # lob off last 3 bytes

tests/debug.py

@@ -4,23 +4,29 @@ import struct
 import binascii
 
 TYPES = {
-    (0x1ff, 0x011): 'create reg',
-    (0x1ff, 0x010): 'create dir',
-    (0x1ff, 0x001): 'superblock',
-    (0x1ff, 0x020): 'delete',
-    (0x1f0, 0x0e0): 'globals',
-    (0x1ff, 0x080): 'tail soft',
-    (0x1ff, 0x081): 'tail hard',
-    (0x1f0, 0x0a0): 'crc',
-    (0x1ff, 0x040): 'struct dir',
-    (0x1ff, 0x041): 'struct inline',
-    (0x1ff, 0x042): 'struct ctz',
-    (0x100, 0x100): 'attr',
+    (0x700, 0x400): 'splice',
+    (0x7ff, 0x401): 'create',
+    (0x7ff, 0x4ff): 'delete',
+    (0x700, 0x000): 'name',
+    (0x7ff, 0x001): 'name reg',
+    (0x7ff, 0x002): 'name dir',
+    (0x7ff, 0x0ff): 'name superblock',
+    (0x700, 0x200): 'struct',
+    (0x7ff, 0x200): 'struct dir',
+    (0x7ff, 0x202): 'struct ctz',
+    (0x7ff, 0x201): 'struct inline',
+    (0x700, 0x300): 'userattr',
+    (0x700, 0x600): 'tail',
+    (0x7ff, 0x600): 'tail soft',
+    (0x7ff, 0x601): 'tail hard',
+    (0x700, 0x700): 'gstate',
+    (0x7ff, 0x7ff): 'gstate move',
+    (0x700, 0x500): 'crc',
 }
 
 def typeof(type):
-    for prefix in range(9):
-        mask = 0x1ff & ~((1 << prefix)-1)
+    for prefix in range(12):
+        mask = 0x7ff & ~((1 << prefix)-1)
         if (mask, type & mask) in TYPES:
             return TYPES[mask, type & mask] + (
                 ' %0*x' % (prefix/4, type & ((1 << prefix)-1))
@@ -59,7 +65,7 @@ def main(*blocks):
     print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True))
 
     # go through each tag, print useful information
-    print "%-4s  %-8s  %-14s  %3s  %3s  %s" % (
+    print "%-4s  %-8s  %-14s  %3s %4s  %s" % (
         'off', 'tag', 'type', 'id', 'len', 'dump')
 
     tag = 0xffffffff
@@ -75,26 +81,26 @@ def main(*blocks):
         tag ^= ntag
         off += 4
 
-        type = (tag & 0x7fc00000) >> 22
-        id   = (tag & 0x003fe000) >> 13
-        size = (tag & 0x00001fff) >> 0
-        iscrc = (type & 0x1f0) == 0x0f0
+        type = (tag & 0x7ff00000) >> 20
+        id   = (tag & 0x000ffc00) >> 10
+        size = (tag & 0x000003ff) >> 0
+        iscrc = (type & 0x700) == 0x500
 
-        data = file.read(size if size != 0x1fff else 0)
+        data = file.read(size if size != 0x3ff else 0)
         if iscrc:
             crc = binascii.crc32(data[:4], crc)
         else:
             crc = binascii.crc32(data, crc)
 
-        print '%04x: %08x  %-14s  %3s  %3s  %-23s  %-8s' % (
+        print '%04x: %08x  %-15s %3s %4s  %-23s  %-8s' % (
             off, tag,
             typeof(type) + (' bad!' if iscrc and ~crc else ''),
-            id if id != 0x1ff else '.',
-            size if size != 0x1fff else 'x',
+            id if id != 0x3ff else '.',
+            size if size != 0x3ff else 'x',
             ' '.join('%02x' % ord(c) for c in data[:8]),
             ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
 
-        off += size if size != 0x1fff else 0
+        off += size if size != 0x3ff else 0
         if iscrc:
             crc = 0
             tag ^= (type & 1) << 31

tests/test_alloc.sh

@@ -253,7 +253,7 @@ tests/test.py << TEST
 
     // find out max file size
     lfs_mkdir(&lfs, "exhaustiondir") => 0;
-    for (int i = 0; i < 9; i++) {
+    for (int i = 0; i < 10; i++) {
         sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
         lfs_mkdir(&lfs, (char*)buffer) => 0;
     }
@@ -275,7 +275,7 @@ tests/test.py << TEST
 
     lfs_remove(&lfs, "exhaustion") => 0;
     lfs_remove(&lfs, "exhaustiondir") => 0;
-    for (int i = 0; i < 9; i++) {
+    for (int i = 0; i < 10; i++) {
         sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
         lfs_remove(&lfs, (char*)buffer) => 0;
     }
@@ -287,7 +287,7 @@ tests/test.py << TEST
     }
     lfs_file_sync(&lfs, &file[0]) => 0;
 
-    for (int i = 0; i < 9; i++) {
+    for (int i = 0; i < 10; i++) {
         sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
         lfs_mkdir(&lfs, (char*)buffer) => 0;
     }

tests/test_format.sh

@@ -12,8 +12,7 @@ TEST
 echo "--- Basic mounting ---"
 tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
-TEST
-tests/test.py << TEST
+
     lfs_mount(&lfs, &cfg) => 0;
     lfs_unmount(&lfs) => 0;
 TEST