Updated .travis.yml and added additional geometry constraints

Moved .travis.yml over to use the new test framework. A part of this
involved testing all of the configurations ran on the old framework
and deciding which to carry over. The new framework duplicates some of
the cases tested by the configurations so some configurations could be
dropped.

The .travis.yml includes some extreme ones, such as no inline files,
relocations every cycle, no intrinsics, power-loss every byte, unaligned
block_count and lookahead, and odd read_sizes.

There were several configurations were some tests failed because of
limitations in the tests themselves, so many conditions were added
to make sure the configurations can run on as many tests as possible.

.travis.yml

@@ -2,6 +2,7 @@
 env:
   global:
     - CFLAGS=-Werror
+    - MAKEFLAGS=-j
 
 # Common test script
 script:
@@ -15,32 +16,27 @@ script:
         -include stdio.h"
 
   # run tests
-  - make test QUIET=1
+  - make clean test TFLAGS+="-nrk"
 
-  # run tests with a few different configurations
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1      -DLFS_CACHE_SIZE=4"
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512    -DLFS_CACHE_SIZE=512 -DLFS_BLOCK_CYCLES=16"
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=8      -DLFS_CACHE_SIZE=16  -DLFS_BLOCK_CYCLES=2"
-  - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256"
+  # common real-life geometries
+  # NOR flash: read/prog = 1 block = 4KiB
+  - make clean test TFLAGS+="-nrk -DLFS_READ_SIZE=1 -DLFS_BLOCK_SIZE=4096"
+  # SD card: read/prog = 512 block = 512
+  - make clean test TFLAGS+="-nrk -DLFS_READ_SIZE=512 -DLFS_BLOCK_SIZE=512"
+  # NAND flash: read/prog = 4KiB block = 32KiB
+  - make clean test TFLAGS+="-nrk -DLFS_READ_SIZE=4096 -DLFS_BLOCK_SIZE=\(32*1024\)"
 
-  - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0"
-  - make clean test QUIET=1 CFLAGS+="-DLFS_EMUBD_ERASE_VALUE=0xff"
-  - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
-
-  # additional configurations that don't support all tests (this should be
-  # fixed but at the moment it is what it is)
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_BLOCK_SIZE=4096"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=\(2*1024\) -DLFS_BLOCK_SIZE=\(64*1024\)"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=\(8*1024\) -DLFS_BLOCK_SIZE=\(64*1024\)"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=11 -DLFS_BLOCK_SIZE=704"
+  # other extreme geometries for corner cases
+  - make clean test TFLAGS+="-nrk -DLFS_NO_INTRINSICS"
+  - make clean test TFLAGS+="-nrk -DLFS_INLINE_MAX=0"
+  - make clean test TFLAGS+="-nrk -DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=1"
+  - make clean test TFLAGS+="-nrk -DLFS_BLOCK_CYCLES=1"
+  - make clean test TFLAGS+="-nrk -DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256"
+  - make clean test TFLAGS+="-nrk -DLFS_READ_SIZE=11 -DLFS_BLOCK_SIZE=704"
 
   # compile and find the code size with the smallest configuration
   - make clean size
-        OBJ="$(ls lfs*.o | tr '\n' ' ')"
+        OBJ="$(ls lfs*.c | sed 's/\.c/\.o/' | tr '\n' ' ')"
         CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR"
         | tee sizes
 
@@ -77,7 +73,7 @@ jobs:
         - STAGE=test
         - NAME=littlefs-arm
         - CC="arm-linux-gnueabi-gcc --static -mthumb"
-        - EXEC="qemu-arm"
+        - TFLAGS="-e qemu-arm"
       install:
         - sudo apt-get install
               gcc-arm-linux-gnueabi
@@ -92,7 +88,7 @@ jobs:
         - STAGE=test
         - NAME=littlefs-powerpc
         - CC="powerpc-linux-gnu-gcc --static"
-        - EXEC="qemu-ppc"
+        - TFLAGS="-e qemu-ppc"
       install:
         - sudo apt-get install
               gcc-powerpc-linux-gnu
@@ -107,7 +103,7 @@ jobs:
         - STAGE=test
         - NAME=littlefs-mips
         - CC="mips-linux-gnu-gcc --static"
-        - EXEC="qemu-mips"
+        - TFLAGS="-e qemu-mips"
       install:
         - sudo apt-get install
               gcc-mips-linux-gnu
@@ -116,6 +112,16 @@ jobs:
         - mips-linux-gnu-gcc --version
         - qemu-mips -version
 
+    # test under valgrind, checking for memory errors
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-valgrind
+        - TFLAGS="--valgrind"
+      install:
+        - sudo apt-get install valgrind
+        - valgrind --version
+
     # self-host with littlefs-fuse for fuzz test
     - stage: test
       env:
@@ -288,6 +294,9 @@ before_install:
             \"description\": \"${STATUS:-In progress}\",
             \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
         }"
+  - sudo apt-get install python3-pip
+  - sudo pip3 install toml
+  
 
 after_failure:
   - |

scripts/test.py

@@ -184,7 +184,8 @@ class TestCase:
         elif self.if_ is not None:
             if_ = self.if_
             while True:
-                for k, v in self.defines.items():
+                for k, v in sorted(self.defines.items(),
+                        key=lambda x: len(x[0]), reverse=True):
                     if k in if_:
                         if_ = if_.replace(k, '(%s)' % v)
                         break
@@ -295,11 +296,11 @@ class ValgrindTestCase(TestCase):
         return not self.leaky and super().shouldtest(**args)
 
     def test(self, exec=[], **args):
-        exec = exec + [
+        exec = [
             'valgrind',
             '--leak-check=full',
             '--error-exitcode=4',
-            '-q']
+            '-q'] + exec
         return super().test(exec=exec, **args)
 
 class ReentrantTestCase(TestCase):
@@ -310,7 +311,7 @@ class ReentrantTestCase(TestCase):
     def shouldtest(self, **args):
         return self.reentrant and super().shouldtest(**args)
 
-    def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
+    def test(self, persist=False, gdb=False, failure=None, **args):
         for cycles in it.count(1):
             # clear disk first?
             if cycles == 1 and persist != 'noerase':
@@ -376,10 +377,11 @@ class TestSuite:
             # code lineno?
             if 'code' in case:
                 case['code_lineno'] = code_linenos.pop()
-            # give our case's config a copy of our "global" config
-            for k, v in config.items():
-                if k not in case:
-                    case[k] = v
+            # merge conditions if necessary
+            if 'if' in config and 'if' in case:
+                case['if'] = '(%s) && (%s)' % (config['if'], case['if'])
+            elif 'if' in config:
+                case['if'] = config['if']
             # initialize test case
             self.cases.append(TestCase(case, filter=filter,
                 suite=self, caseno=i+1, lineno=lineno, **args))

tests/test_alloc.toml

@@ -1,9 +1,10 @@
 # allocator tests
-# note for these to work there are many constraints on the device geometry
+# note for these to work there are a number constraints on the device geometry
+if = 'LFS_BLOCK_CYCLES == -1'
 
 [[case]] # parallel allocation test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
     lfs_file_t files[FILES];
@@ -46,7 +47,7 @@ code = '''
 
 [[case]] # serial allocation test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
 
@@ -85,7 +86,7 @@ code = '''
 
 [[case]] # parallel allocation reuse test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 define.CYCLES = [1, 10]
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
@@ -140,7 +141,7 @@ code = '''
 
 [[case]] # serial allocation reuse test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 define.CYCLES = [1, 10]
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};

tests/test_badblocks.toml

@@ -1,3 +1,6 @@
+# bad blocks with block cycles should be tested in test_relocations
+if = 'LFS_BLOCK_CYCLES == -1'
+
 [[case]] # single bad blocks
 define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster
 define.LFS_ERASE_CYCLES = 0xffffffff

tests/test_entries.toml

@@ -3,7 +3,7 @@
 # still pass with other inline sizes but wouldn't be testing anything.
 
 define.LFS_CACHE_SIZE = 512
-if = 'LFS_CACHE_SIZE == 512'
+if = 'LFS_CACHE_SIZE % LFS_PROG_SIZE == 0 && LFS_CACHE_SIZE == 512'
 
 [[case]] # entry grow test
 code = '''

tests/test_exhaustion.toml

@@ -340,6 +340,7 @@ define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster
 define.LFS_BLOCK_CYCLES = [5, 4, 3, 2, 1]
 define.CYCLES = 100
 define.FILES = 10
+if = 'LFS_BLOCK_CYCLES < CYCLES/10'
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;

tests/test_move.toml

@@ -148,6 +148,7 @@ code = '''
 
 [[case]] # move file corrupt source and dest
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -239,6 +240,7 @@ code = '''
 
 [[case]] # move file after corrupt
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -593,6 +595,7 @@ code = '''
 
 [[case]] # move dir corrupt source and dest
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -692,6 +695,7 @@ code = '''
 
 [[case]] # move dir after corrupt
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;

tests/test_orphans.toml

@@ -1,5 +1,6 @@
 [[case]] # orphan test
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;

tests/test_truncate.toml

@@ -100,7 +100,7 @@ code = '''
     lfs_file_open(&lfs, &file, "sequence",
             LFS_O_RDWR | LFS_O_CREAT | LFS_O_TRUNC) => 0;
 
-    size = lfs.cfg->cache_size;
+    size = lfs_min(lfs.cfg->cache_size, sizeof(buffer)/2);
     lfs_size_t qsize = size / 4;
     uint8_t *wb = buffer;
     uint8_t *rb = buffer + size;