Fixed incorrect documentation in test.py

The argparse documented an outdated format, and was off by 1.

Found by sender6

scripts/test.py

@@ -20,50 +20,19 @@ import pty
 import errno
 import signal
 
-TEST_PATHS = 'tests'
+TESTDIR = 'tests'
 RULES = """
-# add block devices to sources
-TESTSRC ?= $(SRC) $(wildcard bd/*.c)
-
 define FLATTEN
-%(path)s%%$(subst /,.,$(target)): $(target)
+tests/%$(subst /,.,$(target)): $(target)
     ./scripts/explode_asserts.py $$< -o $$@
 endef
-$(foreach target,$(TESTSRC),$(eval $(FLATTEN)))
+$(foreach target,$(SRC),$(eval $(FLATTEN)))
 
--include %(path)s*.d
-.SECONDARY:
+-include tests/*.d
 
-%(path)s.test: %(path)s.test.o \\
-        $(foreach t,$(subst /,.,$(TESTSRC:.c=.o)),%(path)s.$t)
+.SECONDARY:
+%.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.$f)
     $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
-
-# needed in case builddir is different
-%(path)s%%.o: %(path)s%%.c
-    $(CC) -c -MMD $(CFLAGS) $< -o $@
-"""
-COVERAGE_RULES = """
-%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage
-
-# delete lingering coverage
-%(path)s.test: | %(path)s.info.clean
-.PHONY: %(path)s.info.clean
-%(path)s.info.clean:
-    rm -f %(path)s*.gcda
-
-# accumulate coverage info
-.PHONY: %(path)s.info
-%(path)s.info:
-    $(strip $(LCOV) -c \\
-        $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\
-        --rc 'geninfo_adjust_src_path=$(shell pwd)' \\
-        -o $@)
-    $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@
-ifdef COVERAGETARGET
-    $(strip $(LCOV) -a $@ \\
-        $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\
-        -o $(COVERAGETARGET))
-endif
 """
 GLOBALS = """
 //////////////// AUTOGENERATED TEST ////////////////
@@ -150,8 +119,6 @@ class TestCase:
         self.if_ = config.get('if', None)
         self.in_ = config.get('in', None)
 
-        self.result = None
-
     def __str__(self):
         if hasattr(self, 'permno'):
             if any(k not in self.case.defines for k in self.defines):
@@ -212,7 +179,7 @@ class TestCase:
                 len(self.filter) >= 2 and
                 self.filter[1] != self.permno):
             return False
-        elif args.get('no_internal') and self.in_ is not None:
+        elif args.get('no_internal', False) and self.in_ is not None:
             return False
         elif self.if_ is not None:
             if_ = self.if_
@@ -246,7 +213,7 @@ class TestCase:
                 try:
                     with open(disk, 'w') as f:
                         f.truncate(0)
-                    if args.get('verbose'):
+                    if args.get('verbose', False):
                         print('truncate --size=0', disk)
                 except FileNotFoundError:
                     pass
@@ -270,14 +237,14 @@ class TestCase:
                     '-ex', 'r'])
             ncmd.extend(['--args'] + cmd)
 
-            if args.get('verbose'):
+            if args.get('verbose', False):
                 print(' '.join(shlex.quote(c) for c in ncmd))
             signal.signal(signal.SIGINT, signal.SIG_IGN)
             sys.exit(sp.call(ncmd))
 
         # run test case!
         mpty, spty = pty.openpty()
-        if args.get('verbose'):
+        if args.get('verbose', False):
             print(' '.join(shlex.quote(c) for c in cmd))
         proc = sp.Popen(cmd, stdout=spty, stderr=spty)
         os.close(spty)
@@ -293,7 +260,7 @@ class TestCase:
                         break
                     raise
                 stdout.append(line)
-                if args.get('verbose'):
+                if args.get('verbose', False):
                     sys.stdout.write(line)
                 # intercept asserts
                 m = re.match(
@@ -332,7 +299,7 @@ class ValgrindTestCase(TestCase):
         return not self.leaky and super().shouldtest(**args)
 
     def test(self, exec=[], **args):
-        verbose = args.get('verbose')
+        verbose = args.get('verbose', False)
         uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1)
         exec = [
             'valgrind',
@@ -384,17 +351,12 @@ class TestSuite:
         self.name = os.path.basename(path)
         if self.name.endswith('.toml'):
             self.name = self.name[:-len('.toml')]
-        if args.get('build_dir'):
-            self.toml = path
-            self.path = args['build_dir'] + '/' + path
-        else:
-            self.toml = path
-            self.path = path
+        self.path = path
         self.classes = classes
         self.defines = defines.copy()
         self.filter = filter
 
-        with open(self.toml) as f:
+        with open(path) as f:
             # load tests
             config = toml.load(f)
 
@@ -505,7 +467,7 @@ class TestSuite:
 
     def build(self, **args):
         # build test files
-        tf = open(self.path + '.test.tc', 'w')
+        tf = open(self.path + '.test.c.t', 'w')
         tf.write(GLOBALS)
         if self.code is not None:
             tf.write('#line %d "%s"\n' % (self.code_lineno, self.path))
@@ -515,7 +477,7 @@ class TestSuite:
         for case in self.cases:
             if case.in_ not in tfs:
                 tfs[case.in_] = open(self.path+'.'+
-                    re.sub('(\.c)?$', '.tc', case.in_.replace('/', '.')), 'w')
+                    case.in_.replace('/', '.')+'.t', 'w')
                 tfs[case.in_].write('#line 1 "%s"\n' % case.in_)
                 with open(case.in_) as f:
                     for line in f:
@@ -554,33 +516,25 @@ class TestSuite:
 
         # write makefiles
         with open(self.path + '.mk', 'w') as mk:
-            mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path))
+            mk.write(RULES.replace(4*' ', '\t'))
             mk.write('\n')
 
-            # add coverage hooks?
-            if args.get('coverage'):
-                mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict(
-                    path=self.path))
-                mk.write('\n')
-
             # add truely global defines globally
             for k, v in sorted(self.defines.items()):
-                mk.write('%s.test: override CFLAGS += -D%s=%r\n'
-                    % (self.path, k, v))
+                mk.write('%s: override CFLAGS += -D%s=%r\n' % (
+                    self.path+'.test', k, v))
 
             for path in tfs:
                 if path is None:
                     mk.write('%s: %s | %s\n' % (
                         self.path+'.test.c',
-                        self.toml,
-                        self.path+'.test.tc'))
+                        self.path,
+                        self.path+'.test.c.t'))
                 else:
                     mk.write('%s: %s %s | %s\n' % (
                         self.path+'.'+path.replace('/', '.'),
-                        self.toml,
-                        path,
-                        self.path+'.'+re.sub('(\.c)?$', '.tc',
-                            path.replace('/', '.'))))
+                        self.path, path,
+                        self.path+'.'+path.replace('/', '.')+'.t'))
                 mk.write('\t./scripts/explode_asserts.py $| -o $@\n')
 
         self.makefile = self.path + '.mk'
@@ -603,7 +557,7 @@ class TestSuite:
                 if not args.get('verbose', True):
                     sys.stdout.write(FAIL)
                     sys.stdout.flush()
-                if not args.get('keep_going'):
+                if not args.get('keep_going', False):
                     if not args.get('verbose', True):
                         sys.stdout.write('\n')
                     raise
@@ -625,30 +579,30 @@ def main(**args):
 
     # and what class of TestCase to run
     classes = []
-    if args.get('normal'):
+    if args.get('normal', False):
         classes.append(TestCase)
-    if args.get('reentrant'):
+    if args.get('reentrant', False):
         classes.append(ReentrantTestCase)
-    if args.get('valgrind'):
+    if args.get('valgrind', False):
         classes.append(ValgrindTestCase)
     if not classes:
         classes = [TestCase]
 
     suites = []
-    for testpath in args['test_paths']:
+    for testpath in args['testpaths']:
         # optionally specified test case/perm
         testpath, *filter = testpath.split('#')
         filter = [int(f) for f in filter]
 
         # figure out the suite's toml file
         if os.path.isdir(testpath):
-            testpath = testpath + '/*.toml'
+            testpath = testpath + '/test_*.toml'
         elif os.path.isfile(testpath):
             testpath = testpath
         elif testpath.endswith('.toml'):
-            testpath = TEST_PATHS + '/' + testpath
+            testpath = TESTDIR + '/' + testpath
         else:
-            testpath = TEST_PATHS + '/' + testpath + '.toml'
+            testpath = TESTDIR + '/' + testpath + '.toml'
 
         # find tests
         for path in glob.glob(testpath):
@@ -674,7 +628,7 @@ def main(**args):
         list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
         [target for target in targets])
     mpty, spty = pty.openpty()
-    if args.get('verbose'):
+    if args.get('verbose', False):
         print(' '.join(shlex.quote(c) for c in cmd))
     proc = sp.Popen(cmd, stdout=spty, stderr=spty)
     os.close(spty)
@@ -688,14 +642,14 @@ def main(**args):
                 break
             raise
         stdout.append(line)
-        if args.get('verbose'):
+        if args.get('verbose', False):
             sys.stdout.write(line)
         # intercept warnings
         m = re.match(
             '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'
             .format('(?:\033\[[\d;]*.| )*', 'warning'),
             line)
-        if m and not args.get('verbose'):
+        if m and not args.get('verbose', False):
             try:
                 with open(m.group(1)) as f:
                     lineno = int(m.group(2))
@@ -708,26 +662,27 @@ def main(**args):
             except:
                 pass
     proc.wait()
+
     if proc.returncode != 0:
-        if not args.get('verbose'):
+        if not args.get('verbose', False):
             for line in stdout:
                 sys.stdout.write(line)
-        sys.exit(-1)
+        sys.exit(-3)
 
     print('built %d test suites, %d test cases, %d permutations' % (
         len(suites),
         sum(len(suite.cases) for suite in suites),
         sum(len(suite.perms) for suite in suites)))
 
-    total = 0
+    filtered = 0
     for suite in suites:
         for perm in suite.perms:
-            total += perm.shouldtest(**args)
-    if total != sum(len(suite.perms) for suite in suites):
-        print('filtered down to %d permutations' % total)
+            filtered += perm.shouldtest(**args)
+    if filtered != sum(len(suite.perms) for suite in suites):
+        print('filtered down to %d permutations' % filtered)
 
     # only requested to build?
-    if args.get('build'):
+    if args.get('build', False):
         return 0
 
     print('====== testing ======')
@@ -742,12 +697,15 @@ def main(**args):
     failed = 0
     for suite in suites:
         for perm in suite.perms:
+            if not hasattr(perm, 'result'):
+                continue
+
             if perm.result == PASS:
                 passed += 1
-            elif isinstance(perm.result, TestFailure):
+            else:
                 sys.stdout.write(
                     "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m "
-                    "{perm} failed\n".format(
+                    "{perm} failed with {returncode}\n".format(
                         perm=perm, path=perm.suite.path, lineno=perm.lineno,
                         returncode=perm.result.returncode or 0))
                 if perm.result.stdout:
@@ -765,33 +723,11 @@ def main(**args):
                 sys.stdout.write('\n')
                 failed += 1
 
-    if args.get('coverage'):
-        # collect coverage info
-        # why -j1? lcov doesn't work in parallel because of gcov limitations
-        cmd = (['make', '-j1', '-f', 'Makefile'] +
-            list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
-            (['COVERAGETARGET=%s' % args['coverage']]
-                if isinstance(args['coverage'], str) else []) +
-            [suite.path + '.info' for suite in suites
-                if any(perm.result == PASS for perm in suite.perms)])
-        if args.get('verbose'):
-            print(' '.join(shlex.quote(c) for c in cmd))
-        proc = sp.Popen(cmd,
-            stdout=sp.PIPE if not args.get('verbose') else None,
-            stderr=sp.STDOUT if not args.get('verbose') else None,
-            universal_newlines=True)
-        proc.wait()
-        if proc.returncode != 0:
-            if not args.get('verbose'):
-                for line in proc.stdout:
-                    sys.stdout.write(line)
-            sys.exit(-1)
-
-    if args.get('gdb'):
+    if args.get('gdb', False):
         failure = None
         for suite in suites:
             for perm in suite.perms:
-                if isinstance(perm.result, TestFailure):
+                if getattr(perm, 'result', PASS) != PASS:
                     failure = perm.result
         if failure is not None:
             print('======= gdb ======')
@@ -799,22 +735,20 @@ def main(**args):
             failure.case.test(failure=failure, **args)
             sys.exit(0)
 
-    print('tests passed %d/%d (%.2f%%)' % (passed, total,
-        100*(passed/total if total else 1.0)))
-    print('tests failed %d/%d (%.2f%%)' % (failed, total,
-        100*(failed/total if total else 1.0)))
+    print('tests passed: %d' % passed)
+    print('tests failed: %d' % failed)
     return 1 if failed > 0 else 0
 
 if __name__ == "__main__":
     import argparse
     parser = argparse.ArgumentParser(
         description="Run parameterized tests in various configurations.")
-    parser.add_argument('test_paths', nargs='*', default=[TEST_PATHS],
+    parser.add_argument('testpaths', nargs='*', default=[TESTDIR],
         help="Description of test(s) to run. By default, this is all tests \
             found in the \"{0}\" directory. Here, you can specify a different \
-            directory of tests, a specific file, a suite by name, and even a \
-            specific test case by adding brackets. For example \
-            \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TEST_PATHS))
+            directory of tests, a specific file, a suite by name, and even \
+            specific test cases and permutations. For example \
+            \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TESTDIR))
     parser.add_argument('-D', action='append', default=[],
         help="Overriding parameter definitions.")
     parser.add_argument('-v', '--verbose', action='store_true',
@@ -835,19 +769,10 @@ if __name__ == "__main__":
         help="Run tests normally.")
     parser.add_argument('-r', '--reentrant', action='store_true',
         help="Run reentrant tests with simulated power-loss.")
-    parser.add_argument('--valgrind', action='store_true',
+    parser.add_argument('-V', '--valgrind', action='store_true',
         help="Run non-leaky tests under valgrind to check for memory leaks.")
-    parser.add_argument('--exec', default=[], type=lambda e: e.split(),
+    parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(' '),
         help="Run tests with another executable prefixed on the command line.")
-    parser.add_argument('--disk',
+    parser.add_argument('-d', '--disk',
         help="Specify a file to use for persistent/reentrant tests.")
-    parser.add_argument('--coverage', type=lambda x: x if x else True,
-        nargs='?', const='',
-        help="Collect coverage information during testing. This uses lcov/gcov \
-            to accumulate coverage information into *.info files. May also \
-            a path to a *.info file to accumulate coverage info into.")
-    parser.add_argument('--build-dir',
-        help="Build relative to the specified directory instead of the \
-            current directory.")
-
     sys.exit(main(**vars(parser.parse_args())))