Reworked test defines a bit to use one common array layout

Previously didn't think this would work without making test.py aware of
the number of implicit defines, which risks being incredibly fragile.
Fortunately it turns out we can defer the actual array size calculation
until the C preprocessor. This simplifies a few things.

Also a bitmap-based caching layer for the defines. Since the test
defines have been upgraded to callbacks recursive defines risk spending
a decent amount of time evaluating on every lookup. Some quick testing
shows 408015154 hits to 46160 misses so that's a good sign.

Also changed the geometries to be their own leb16-encoded part of the
test identifier. This means any geometry can be captured and reproduced
with just the test identifier. Here are the current test geometries:

./runners/test_runner --list-geometries
geometry                    read    prog   erase   count        size  leb16
d,default                     16      16     512    2048     1048576  g1gg2
e,eeprom                       1       1     512    2048     1048576  1gg2
E,emmc                       512     512     512    2048     1048576  gg2
n,nor                          1       1    4096     256     1048576  1ggg1
N,nand                      4096    4096   32768      32     1048576  ggg1ggg8

runners/test_runner.h

@@ -38,7 +38,7 @@ struct test_case {
     test_flags_t flags;
     size_t permutations;
 
-    intmax_t (*const *const *defines)(void);
+    intmax_t (*const *const *defines)(size_t);
 
     bool (*filter)(void);
     void (*run)(struct lfs_config *cfg);
@@ -59,60 +59,50 @@ struct test_suite {
 
 
 // access generated test defines
-intmax_t test_predefine(size_t define);
+//intmax_t test_predefine(size_t define);
 intmax_t test_define(size_t define);
 
 // a few preconfigured defines that control how tests run
-#define READ_SIZE           test_predefine(0)
-#define PROG_SIZE           test_predefine(1)
-#define BLOCK_SIZE          test_predefine(2)
-#define BLOCK_COUNT         test_predefine(3)
-#define CACHE_SIZE          test_predefine(4)
-#define LOOKAHEAD_SIZE      test_predefine(5)
-#define BLOCK_CYCLES        test_predefine(6)
-#define ERASE_VALUE         test_predefine(7)
-#define ERASE_CYCLES        test_predefine(8)
-#define BADBLOCK_BEHAVIOR   test_predefine(9)
-#define POWERLOSS_BEHAVIOR  test_predefine(10)
-
-#define TEST_PREDEFINE_NAMES { \
-    "READ_SIZE", \
-    "PROG_SIZE", \
-    "BLOCK_SIZE", \
-    "BLOCK_COUNT", \
-    "CACHE_SIZE", \
-    "LOOKAHEAD_SIZE", \
-    "BLOCK_CYCLES", \
-    "ERASE_VALUE", \
-    "ERASE_CYCLES", \
-    "BADBLOCK_BEHAVIOR", \
-    "POWERLOSS_BEHAVIOR", \
-}
-#define TEST_PREDEFINE_COUNT 11
-
-
-// default predefines
-#define TEST_DEFAULTS { \
-    /* LOOKAHEAD_SIZE */     16, \
-    /* BLOCK_CYCLES */       -1, \
-    /* ERASE_VALUE */        0xff, \
-    /* ERASE_CYCLES */       0, \
-    /* BADBLOCK_BEHAVIOR */  LFS_TESTBD_BADBLOCK_PROGERROR, \
-    /* POWERLOSS_BEHAVIOR */ LFS_TESTBD_POWERLOSS_NOOP, \
-}
-#define TEST_DEFAULT_DEFINE_COUNT 5
-
-// test geometries
-#define TEST_GEOMETRIES { \
-    /*geometry, read, write,    erase,                 count, cache */ \
-    {"test",   {  16,    16,      512,       (1024*1024)/512,    64}}, \
-    {"eeprom", {   1,     1,      512,       (1024*1024)/512,    64}}, \
-    {"emmc",   { 512,   512,      512,       (1024*1024)/512,   512}}, \
-    {"nor",    {   1,     1,     4096,      (1024*1024)/4096,    64}}, \
-    {"nand",   {4096,  4096,  32*1024, (1024*1024)/(32*1024),  4096}}, \
-}
-#define TEST_GEOMETRY_COUNT 5
-#define TEST_GEOMETRY_DEFINE_COUNT 5
+ 
+#define READ_SIZE_i          0
+#define PROG_SIZE_i          1
+#define BLOCK_SIZE_i         2
+#define BLOCK_COUNT_i        3
+#define CACHE_SIZE_i         4
+#define LOOKAHEAD_SIZE_i     5
+#define BLOCK_CYCLES_i       6
+#define ERASE_VALUE_i        7
+#define ERASE_CYCLES_i       8
+#define BADBLOCK_BEHAVIOR_i  9
+#define POWERLOSS_BEHAVIOR_i 10
+
+#define READ_SIZE           test_define(READ_SIZE_i)
+#define PROG_SIZE           test_define(PROG_SIZE_i)
+#define BLOCK_SIZE          test_define(BLOCK_SIZE_i)
+#define BLOCK_COUNT         test_define(BLOCK_COUNT_i)
+#define CACHE_SIZE          test_define(CACHE_SIZE_i)
+#define LOOKAHEAD_SIZE      test_define(LOOKAHEAD_SIZE_i)
+#define BLOCK_CYCLES        test_define(BLOCK_CYCLES_i)
+#define ERASE_VALUE         test_define(ERASE_VALUE_i)
+#define ERASE_CYCLES        test_define(ERASE_CYCLES_i)
+#define BADBLOCK_BEHAVIOR   test_define(BADBLOCK_BEHAVIOR_i)
+#define POWERLOSS_BEHAVIOR  test_define(POWERLOSS_BEHAVIOR_i)
+
+#define TEST_IMPLICIT_DEFINES \
+    TEST_DEFINE(READ_SIZE,          test_geometry->read_size) \
+    TEST_DEFINE(PROG_SIZE,          test_geometry->prog_size) \
+    TEST_DEFINE(BLOCK_SIZE,         test_geometry->block_size) \
+    TEST_DEFINE(BLOCK_COUNT,        test_geometry->block_count) \
+    TEST_DEFINE(CACHE_SIZE,         lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \
+    TEST_DEFINE(LOOKAHEAD_SIZE,     16) \
+    TEST_DEFINE(BLOCK_CYCLES,       -1) \
+    TEST_DEFINE(ERASE_VALUE,        0xff) \
+    TEST_DEFINE(ERASE_CYCLES,       0) \
+    TEST_DEFINE(BADBLOCK_BEHAVIOR,  LFS_TESTBD_BADBLOCK_PROGERROR) \
+    TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP)
+
+#define TEST_GEOMETRY_DEFINE_COUNT 4
+#define TEST_IMPLICIT_DEFINE_COUNT 11
 
 
 #endif

scripts/test.py

@@ -252,6 +252,7 @@ def compile(**args):
 
             # include test_runner.h in every generated file
             f.writeln("#include \"%s\"" % HEADER_PATH)
+            f.writeln()
 
             # write out generated functions, this can end up in different
             # files depending on the "in" attribute
@@ -270,20 +271,22 @@ def compile(**args):
                                     name = ('__test__%s__%s__%s__%d'
                                         % (suite.name, case.name, k, i))
                                     define_cbs[v] = name
-                                    f.writeln('intmax_t %s(void) {' % name)
+                                    f.writeln('intmax_t %s('
+                                        '__attribute__((unused)) '
+                                        'size_t define) {' % name)
                                     f.writeln(4*' '+'return %s;' % v)
                                     f.writeln('}')
                                     f.writeln()
                         f.writeln('intmax_t (*const *const '
-                            '__test__%s__%s__defines[])(void) = {'
+                            '__test__%s__%s__defines[])(size_t) = {'
                             % (suite.name, case.name))
                         for defines in case.permutations:
-                            f.writeln(4*' '+'(intmax_t (*const[])(void)){')
-                            for define in sorted(suite.defines):
-                                f.writeln(8*' '+'%s,' % (
-                                    define_cbs[defines[define]]
-                                        if define in defines
-                                        else 'NULL'))
+                            f.writeln(4*' '+'(intmax_t (*const['
+                                'TEST_IMPLICIT_DEFINE_COUNT+%d])(size_t)){' % (
+                                len(suite.defines)))
+                            for k, v in sorted(defines.items()):
+                                f.writeln(8*' '+'[%-24s] = %s,' % (
+                                    k+'_i', define_cbs[v]))
                             f.writeln(4*' '+'},')
                         f.writeln('};')
                         f.writeln()    
@@ -328,8 +331,10 @@ def compile(**args):
                 if suite.defines:
                     for i, define in enumerate(sorted(suite.defines)):
                         f.writeln('#ifndef %s' % define)
-                        f.writeln('#define %-24s test_define(%d)'
-                            % (define, i))
+                        f.writeln('#define %-24s '
+                            'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i))
+                        f.writeln('#define %-24s '
+                            'test_define(%s)' % (define, define+'_i'))
                         f.writeln('#endif')
                     f.writeln()
 
@@ -340,7 +345,7 @@ def compile(**args):
                     else:
                         if case.defines:
                             f.writeln('extern intmax_t (*const *const '
-                                '__test__%s__%s__defines[])(void);'
+                                '__test__%s__%s__defines[])(size_t);'
                                 % (suite.name, case.name))
                         if suite.if_ is not None or case.if_ is not None:
                             f.writeln('extern bool __test__%s__%s__filter('
@@ -364,11 +369,14 @@ def compile(**args):
                         or 0))
                 if suite.defines:
                     # create suite define names
-                    f.writeln(4*' '+'.define_names = (const char *const[]){')
+                    f.writeln(4*' '+'.define_names = (const char *const['
+                        'TEST_IMPLICIT_DEFINE_COUNT+%d]){' % (
+                        len(suite.defines)))
                     for k in sorted(suite.defines):
-                        f.writeln(8*' '+'"%s",' % k)
+                        f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
                     f.writeln(4*' '+'},')
-                f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
+                    f.writeln(4*' '+'.define_count = '
+                        'TEST_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
                 f.writeln(4*' '+'.cases = (const struct test_case[]){')
                 for case in suite.cases:
                     # create case structs
@@ -415,10 +423,15 @@ def compile(**args):
                                 for i, define in enumerate(
                                         sorted(suite.defines)):
                                     f.writeln('#ifndef %s' % define)
-                                    f.writeln('#define %-24s test_define(%d)'
-                                        % (define, i))
-                                    f.writeln('#define __TEST__%s__NEEDS_UNDEF'
-                                        % define)
+                                    f.writeln('#define %-24s '
+                                        'TEST_IMPLICIT_DEFINE_COUNT+%d' % (
+                                        define+'_i', i))
+                                    f.writeln('#define %-24s '
+                                        'test_define(%s)' % (
+                                        define, define+'_i'))
+                                    f.writeln('#define '
+                                        '__TEST__%s__NEEDS_UNDEF' % (
+                                        define))
                                     f.writeln('#endif')
                                 f.writeln()
 
@@ -431,6 +444,7 @@ def compile(**args):
                                     f.writeln('#undef __TEST__%s__NEEDS_UNDEF'
                                         % define)
                                     f.writeln('#undef %s' % define)
+                                    f.writeln('#undef %s' % (define+'_i'))
                                     f.writeln('#endif')
                                 f.writeln()
 
@@ -469,9 +483,10 @@ def list_(**args):
     if args.get('summary'):          cmd.append('--summary')
     if args.get('list_suites'):      cmd.append('--list-suites')
     if args.get('list_cases'):       cmd.append('--list-cases')
-    if args.get('list_paths'):       cmd.append('--list-paths')
+    if args.get('list_suite_paths'): cmd.append('--list-suite-paths')
+    if args.get('list_case_paths'):  cmd.append('--list-case-paths')
     if args.get('list_defines'):     cmd.append('--list-defines')
-    if args.get('list_defaults'):    cmd.append('--list-defaults')
+    if args.get('list_implicit'):    cmd.append('--list-implicit')
     if args.get('list_geometries'):  cmd.append('--list-geometries')
     if args.get('list_powerlosses'): cmd.append('--list-powerlosses')
 
@@ -503,10 +518,11 @@ def find_cases(runner_, **args):
         m = pattern.match(line)
         if m:
             filtered = int(m.group('filtered'))
+            perms = int(m.group('perms'))
             expected_suite_perms[m.group('suite')] += filtered
             expected_case_perms[m.group('id')] += filtered
             expected_perms += filtered
-            total_perms += int(m.group('perms'))
+            total_perms += perms
     proc.wait()
     if proc.returncode != 0:
         if not args.get('verbose'):
@@ -520,9 +536,9 @@ def find_cases(runner_, **args):
         expected_perms,
         total_perms)
 
-def find_paths(runner_, **args):
+def find_path(runner_, id, **args):
     # query from runner
-    cmd = runner_ + ['--list-paths']
+    cmd = runner_ + ['--list-case-paths', id]
     if args.get('verbose'):
         print(' '.join(shlex.quote(c) for c in cmd))
     proc = sp.Popen(cmd,
@@ -531,14 +547,45 @@ def find_paths(runner_, **args):
         universal_newlines=True,
         errors='replace',
         close_fds=False)
-    paths = co.OrderedDict()
+    path = None
     pattern = re.compile(
         '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
             '(?P<path>[^:]+):(?P<lineno>\d+)')
+    # skip the first line
+    for line in it.islice(proc.stdout, 1, None):
+        m = pattern.match(line)
+        if m and path is None:
+            path_ = m.group('path')
+            lineno = int(m.group('lineno'))
+            path = (path_, lineno)
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    return path
+
+def find_defines(runner_, id, **args):
+    # query implicit defines from runner
+    cmd = runner_ + ['--list-implicit']
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    implicit_defines = co.OrderedDict()
+    pattern = re.compile('^(?P<define>\w+)=(?P<values>.+)')
     for line in proc.stdout:
         m = pattern.match(line)
         if m:
-            paths[m.group('id')] = (m.group('path'), int(m.group('lineno')))
+            define = m.group('define')
+            values = m.group('values').split(',')
+            implicit_defines[define] = set(values)
     proc.wait()
     if proc.returncode != 0:
         if not args.get('verbose'):
@@ -546,11 +593,8 @@ def find_paths(runner_, **args):
                 sys.stdout.write(line)
         sys.exit(-1)
 
-    return paths
-
-def find_defines(runner_, **args):
-    # query from runner
-    cmd = runner_ + ['--list-defines']
+    # query case defines from runner
+    cmd = runner_ + ['--list-defines', id]
     if args.get('verbose'):
         print(' '.join(shlex.quote(c) for c in cmd))
     proc = sp.Popen(cmd,
@@ -560,14 +604,15 @@ def find_defines(runner_, **args):
         errors='replace',
         close_fds=False)
     defines = co.OrderedDict()
-    pattern = re.compile(
-        '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
-            '(?P<defines>(?:\w+=\w+\s*)+)')
+    pattern = re.compile('^(?P<define>\w+)=(?P<value>.+)')
     for line in proc.stdout:
         m = pattern.match(line)
         if m:
-            defines[m.group('id')] = {k: v
-                for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))}
+            define = m.group('define')
+            value = m.group('value')
+            if (define not in implicit_defines
+                    or value not in implicit_defines[define]):
+                defines[define] = value
     proc.wait()
     if proc.returncode != 0:
         if not args.get('verbose'):
@@ -887,16 +932,16 @@ def run(**args):
     print()
 
     # print each failure
-    if failures:
+    for failure in failures:
+        assert failure.id is not None, '%s broken? %r' % (
+            ' '.join(shlex.quote(c) for c in runner_),
+            failure)
+
         # get some extra info from runner
-        runner_paths = find_paths(runner_, **args)
-        runner_defines = find_defines(runner_, **args)
+        path, lineno = find_path(runner_, failure.id, **args)
+        defines = find_defines(runner_, failure.id, **args)
 
-    for failure in failures:
         # show summary of failure
-        path, lineno = runner_paths[testcase(failure.id)]
-        defines = runner_defines.get(failure.id, {})
-
         print('%s%s:%d:%sfailure:%s %s%s failed' % (
             '\x1b[01m' if color(**args) else '',
             path, lineno,
@@ -969,9 +1014,10 @@ def main(**args):
     elif (args.get('summary')
             or args.get('list_suites')
             or args.get('list_cases')
-            or args.get('list_paths')
+            or args.get('list_suite_paths')
+            or args.get('list_case_paths')
             or args.get('list_defines')
-            or args.get('list_defaults')
+            or args.get('list_implicit')
             or args.get('list_geometries')
             or args.get('list_powerlosses')):
         list_(**args)
@@ -1005,20 +1051,23 @@ if __name__ == "__main__":
         help="List test suites.")
     test_parser.add_argument('-L', '--list-cases', action='store_true',
         help="List test cases.")
-    test_parser.add_argument('--list-paths', action='store_true',
-        help="List the path for each test case.")
+    test_parser.add_argument('--list-suite-paths', action='store_true',
+        help="List the path for each test suite.")
+    test_parser.add_argument('--list-case-paths', action='store_true',
+        help="List the path and line number for each test case.")
     test_parser.add_argument('--list-defines', action='store_true',
-        help="List the defines for each test permutation.")
-    test_parser.add_argument('--list-defaults', action='store_true',
-        help="List the default defines in this test-runner.")
+        help="List all defines in this test-runner.")
+    test_parser.add_argument('--list-implicit', action='store_true',
+        help="List implicit defines in this test-runner.")
     test_parser.add_argument('--list-geometries', action='store_true',
-        help="List the disk geometries used for testing.")
+        help="List the available disk geometries.")
     test_parser.add_argument('--list-powerlosses', action='store_true',
         help="List the available power-loss scenarios.")
     test_parser.add_argument('-D', '--define', action='append',
         help="Override a test define.")
-    test_parser.add_argument('-G', '--geometry',
-        help="Filter by geometry.")
+    test_parser.add_argument('-g', '--geometry',
+        help="Comma-separated list of disk geometries to test. \
+            Defaults to d,e,E,n,N.")
     test_parser.add_argument('-p', '--powerloss',
         help="Comma-separated list of power-loss scenarios to test. \
             Defaults to 0,l.")