Continued progress toward feature parity with new test-runner

- Expanded test defines to allow for lists of configurations

  These are useful for changing multi-dimensional test configurations
  without leading to extremely large and less useful configuration
  combinations.

- Made warnings more visible durring test parsing

- Add lfs_testbd.h to implicit test includes

- Fixed issue with not closing files in ./scripts/explode_asserts.py

- Add `make test_runner` and `make test_list` build rules for
  convenience

Makefile

@@ -119,9 +119,16 @@ test:
 test%: tests/test$$(firstword $$(subst \#, ,%)).toml
 	./scripts/test.py $@ $(TESTFLAGS)
 
+.PHONY: test_runner
+test_runner: $(BUILDDIR)runners/test_runner
+
 .PHONY: test_
-test_: $(BUILDDIR)runners/test_runner
-	./scripts/test_.py --runner=$< $(TESTFLAGS_)
+test_: test_runner
+	./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_)
+
+.PHONY: test_list
+test_list: test_runner
+	./scripts/test_.py --runner=$(BUILDDIR)runners/test_runner $(TESTFLAGS_) -l
 
 .PHONY: code
 code: $(OBJ)

scripts/explode_asserts.py

@@ -134,6 +134,15 @@ TYPE = {
     }
 }
 
+def openio(path, mode='r'):
+    if path == '-':
+        if 'r' in mode:
+            return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
+        else:
+            return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
+    else:
+        return open(path, mode)
+
 def mkdecls(outf, maxwidth=16):
     outf.write("#include <stdio.h>\n")
     outf.write("#include <stdbool.h>\n")
@@ -341,32 +350,31 @@ def pstmt(p):
 
 
 def main(args):
-    inf = open(args.input, 'r') if args.input else sys.stdin
-    outf = open(args.output, 'w') if args.output else sys.stdout
-
-    lexemes = LEX.copy()
-    if args.pattern:
-        lexemes['assert'] = args.pattern
-    p = Parse(inf, lexemes)
-
-    # write extra verbose asserts
-    mkdecls(outf, maxwidth=args.maxwidth)
-    if args.input:
-        outf.write("#line %d \"%s\"\n" % (1, args.input))
-
-    # parse and write out stmt at a time
-    try:
-        while True:
-            outf.write(pstmt(p))
-            if p.accept('sep'):
-                outf.write(p.m)
-            else:
-                break
-    except ParseFailure as f:
-        pass
-
-    for i in range(p.off, len(p.tokens)):
-        outf.write(p.tokens[i][1])
+    with openio(args.input or '-', 'r') as inf:
+        with openio(args.output or '-', 'w') as outf:
+            lexemes = LEX.copy()
+            if args.pattern:
+                lexemes['assert'] = args.pattern
+            p = Parse(inf, lexemes)
+
+            # write extra verbose asserts
+            mkdecls(outf, maxwidth=args.maxwidth)
+            if args.input:
+                outf.write("#line %d \"%s\"\n" % (1, args.input))
+
+            # parse and write out stmt at a time
+            try:
+                while True:
+                    outf.write(pstmt(p))
+                    if p.accept('sep'):
+                        outf.write(p.m)
+                    else:
+                        break
+            except ParseFailure as f:
+                pass
+
+            for i in range(p.off, len(p.tokens)):
+                outf.write(p.tokens[i][1])
 
 if __name__ == "__main__":
     import argparse

scripts/test_.py

@@ -24,6 +24,7 @@ RUNNER_PATH = './runners/test_runner'
 
 SUITE_PROLOGUE = """
 #include "runners/test_runner.h"
+#include "bd/lfs_testbd.h"
 #include <stdio.h>
 """
 CASE_PROLOGUE = """
@@ -78,20 +79,30 @@ class TestCase:
         self.valgrind = config.pop('valgrind',
                 config.pop('suite_valgrind', True))
 
-        # figure out defines and the number of resulting permutations
-        self.defines = {}
-        for k, v in (
-                config.pop('suite_defines', {})
-                | config.pop('defines', {})).items():
-            if not isinstance(v, list):
-                v = [v]
-
-            self.defines[k] = v
-
-        self.permutations = m.prod(len(v) for v in self.defines.values())
+        # figure out defines and build possible permutations
+        self.defines = set()
+        self.permutations = []
+
+        suite_defines = config.pop('suite_defines', {})
+        if not isinstance(suite_defines, list):
+            suite_defines = [suite_defines]
+        defines = config.pop('defines', {})
+        if not isinstance(defines, list):
+            defines = [defines]
+
+        # build possible permutations
+        for suite_defines_ in suite_defines:
+            self.defines |= suite_defines_.keys()
+            for defines_ in defines:
+                self.defines |= defines_.keys()
+                self.permutations.extend(map(dict, it.product(*(
+                    [(k, v) for v in (vs if isinstance(vs, list) else [vs])]
+                    for k, vs in sorted(
+                        (suite_defines_ | defines_).items())))))
 
         for k in config.keys():
-            print('warning: in %s, found unused key %r' % (self.id(), k),
+            print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
+                % (self.id(), k),
                 file=sys.stderr)
 
     def id(self):
@@ -130,7 +141,7 @@ class TestSuite:
             # sort in case toml parsing did not retain order
             case_linenos.sort()
 
-            cases = config.pop('cases', [])
+            cases = config.pop('cases')
             for (lineno, name), (nlineno, _) in it.zip_longest(
                     case_linenos, case_linenos[1:],
                     fillvalue=(float('inf'), None)):
@@ -179,8 +190,8 @@ class TestSuite:
                     **case}))
 
             # combine per-case defines
-            self.defines = sorted(
-                set.union(*(set(case.defines) for case in self.cases)))
+            self.defines = set.union(*(
+                set(case.defines) for case in self.cases))
 
             # combine other per-case things
             self.normal = any(case.normal for case in self.cases)
@@ -188,7 +199,8 @@ class TestSuite:
             self.valgrind = any(case.valgrind for case in self.cases)
 
         for k in config.keys():
-            print('warning: in %s, found unused key %r' % (self.id(), k),
+            print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
+                % (self.id(), k),
                 file=sys.stderr)
 
     def id(self):
@@ -266,7 +278,7 @@ def compile(**args):
                             % (f.lineno+1, args['output']))
                     f.writeln()
 
-                for i, define in enumerate(suite.defines):
+                for i, define in enumerate(sorted(suite.defines)):
                     f.writeln('#ifndef %s' % define)
                     f.writeln('#define %-24s test_define(%d)' % (define, i))
                     f.writeln('#endif')
@@ -275,16 +287,13 @@ def compile(**args):
                 for case in suite.cases:
                     # create case defines
                     if case.defines:
-                        sorted_defines = sorted(case.defines.items())
-
                         f.writeln('const test_define_t *const '
                             '__test__%s__%s__defines[] = {'
                             % (suite.name, case.name))
-                        for defines in it.product(*(
-                                [(k, v) for v in vs]
-                                for k, vs in sorted_defines)):
+                        for permutation in case.permutations:
                             f.writeln(4*' '+'(const test_define_t[]){%s},'
-                                % ', '.join('%s' % v for _, v in defines)) 
+                                % ', '.join(str(v) for _, v in sorted(
+                                    permutation.items())))
                         f.writeln('};')
                         f.writeln()
 
@@ -293,9 +302,9 @@ def compile(**args):
                             % (suite.name, case.name))
                         f.writeln(4*' '+'%s,'
                             % ', '.join(
-                                '%s' % [k for k, _ in sorted_defines].index(k)
+                                str(sorted(case.defines).index(k))
                                 if k in case.defines else '0xff'
-                                for k in suite.defines))
+                                for k in sorted(suite.defines)))
                         f.writeln('};')
                         f.writeln()
 
@@ -365,7 +374,8 @@ def compile(**args):
                             'TEST_NORMAL' if case.normal else None,
                             'TEST_REENTRANT' if case.reentrant else None,
                             'TEST_VALGRIND' if case.valgrind else None])))
-                    f.writeln(4*' '+'.permutations = %d,' % case.permutations)
+                    f.writeln(4*' '+'.permutations = %d,'
+                        % len(case.permutations))
                     if case.defines:
                         f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
                             % (suite.name, case.name))
@@ -381,12 +391,13 @@ def compile(**args):
                     f.writeln()
 
                 # create suite define names
-                f.writeln('const char *const __test__%s__define_names[] = {'
-                    % suite.name)
-                for k in suite.defines:
-                    f.writeln(4*' '+'"%s",' % k)
-                f.writeln('};')
-                f.writeln()
+                if suite.defines:
+                    f.writeln('const char *const __test__%s__define_names[] = {'
+                        % suite.name)
+                    for k in sorted(suite.defines):
+                        f.writeln(4*' '+'"%s",' % k)
+                    f.writeln('};')
+                    f.writeln()
 
                 # create suite struct
                 f.writeln('const struct test_suite __test__%s__suite = {'
@@ -399,8 +410,9 @@ def compile(**args):
                         'TEST_NORMAL' if suite.normal else None,
                         'TEST_REENTRANT' if suite.reentrant else None,
                         'TEST_VALGRIND' if suite.valgrind else None])))
-                f.writeln(4*' '+'.define_names = __test__%s__define_names,'
-                    % suite.name)
+                if suite.defines:
+                    f.writeln(4*' '+'.define_names = __test__%s__define_names,'
+                        % suite.name)
                 f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
                 f.writeln(4*' '+'.cases = (const struct test_case *const []){')
                 for case in suite.cases:
@@ -726,7 +738,7 @@ def run_stage(name, runner_, **args):
 
             if not args.get('verbose'):
                 sys.stdout.write('\r\x1b[K'
-                    'running \x1b[%dm%s\x1b[m: '
+                    'running \x1b[%dm%s:\x1b[m '
                     '%d/%d suites, %d/%d cases, %d/%d perms%s '
                     % (32 if not failures else 31,
                         name,
@@ -779,12 +791,12 @@ def run(**args):
     expected = 0
     passed = 0
     failures = []
-    if args.get('by_suites'):
+    if args.get('by_cases'):
         for type in ['normal', 'reentrant', 'valgrind']:
-            for suite in expected_suite_perms.keys():
+            for case in expected_case_perms.keys():
                 expected_, passed_, failures_, killed = run_stage(
-                    '%s %s' % (type, suite),
-                    runner_ + ['--%s' % type, suite],
+                    '%s %s' % (type, case),
+                    runner_ + ['--%s' % type, case],
                     **args)
                 expected += expected_
                 passed += passed_
@@ -793,12 +805,12 @@ def run(**args):
                     break
             if (failures and not args.get('keep_going')) or killed:
                 break
-    elif args.get('by_cases'):
+    elif args.get('by_suites'):
         for type in ['normal', 'reentrant', 'valgrind']:
-            for case in expected_case_perms.keys():
+            for suite in expected_suite_perms.keys():
                 expected_, passed_, failures_, killed = run_stage(
-                    '%s %s' % (type, case),
-                    runner_ + ['--%s' % type, case],
+                    '%s %s' % (type, suite),
+                    runner_ + ['--%s' % type, suite],
                     **args)
                 expected += expected_
                 passed += passed_
@@ -821,7 +833,7 @@ def run(**args):
 
     # show summary
     print()
-    print('\x1b[%dmdone\x1b[m: %d/%d passed, %d/%d failed, in %.2fs'
+    print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs'
         % (32 if not failures else 31,
             passed, expected, len(failures), expected,
             time.time()-start))