|
@@ -1,860 +1,1027 @@
|
|
|
#!/usr/bin/env python3
|
|
#!/usr/bin/env python3
|
|
|
-
|
|
|
|
|
-# This script manages littlefs tests, which are configured with
|
|
|
|
|
-# .toml files stored in the tests directory.
|
|
|
|
|
|
|
+#
|
|
|
|
|
+# Script to compile and runs tests.
|
|
|
#
|
|
#
|
|
|
|
|
|
|
|
-import toml
|
|
|
|
|
|
|
+import collections as co
|
|
|
|
|
+import errno
|
|
|
import glob
|
|
import glob
|
|
|
-import re
|
|
|
|
|
-import os
|
|
|
|
|
-import io
|
|
|
|
|
import itertools as it
|
|
import itertools as it
|
|
|
-import collections.abc as abc
|
|
|
|
|
-import subprocess as sp
|
|
|
|
|
-import base64
|
|
|
|
|
-import sys
|
|
|
|
|
-import copy
|
|
|
|
|
-import shlex
|
|
|
|
|
|
|
+import math as m
|
|
|
|
|
+import os
|
|
|
import pty
|
|
import pty
|
|
|
-import errno
|
|
|
|
|
|
|
+import re
|
|
|
|
|
+import shlex
|
|
|
|
|
+import shutil
|
|
|
import signal
|
|
import signal
|
|
|
|
|
+import subprocess as sp
|
|
|
|
|
+import threading as th
|
|
|
|
|
+import time
|
|
|
|
|
+import toml
|
|
|
|
|
|
|
|
-TEST_PATHS = 'tests'
|
|
|
|
|
-RULES = """
|
|
|
|
|
-# add block devices to sources
|
|
|
|
|
-TESTSRC ?= $(SRC) $(wildcard bd/*.c)
|
|
|
|
|
-
|
|
|
|
|
-define FLATTEN
|
|
|
|
|
-%(path)s%%$(subst /,.,$(target)): $(target)
|
|
|
|
|
- ./scripts/explode_asserts.py $$< -o $$@
|
|
|
|
|
-endef
|
|
|
|
|
-$(foreach target,$(TESTSRC),$(eval $(FLATTEN)))
|
|
|
|
|
-
|
|
|
|
|
--include %(path)s*.d
|
|
|
|
|
-.SECONDARY:
|
|
|
|
|
|
|
|
|
|
-%(path)s.test: %(path)s.test.o \\
|
|
|
|
|
- $(foreach t,$(subst /,.,$(TESTSRC:.c=.o)),%(path)s.$t)
|
|
|
|
|
- $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
|
|
|
|
|
|
|
+TEST_PATHS = ['tests']
|
|
|
|
|
+RUNNER_PATH = './runners/test_runner'
|
|
|
|
|
|
|
|
-# needed in case builddir is different
|
|
|
|
|
-%(path)s%%.o: %(path)s%%.c
|
|
|
|
|
- $(CC) -c -MMD $(CFLAGS) $< -o $@
|
|
|
|
|
-"""
|
|
|
|
|
-COVERAGE_RULES = """
|
|
|
|
|
-%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage
|
|
|
|
|
-
|
|
|
|
|
-# delete lingering coverage
|
|
|
|
|
-%(path)s.test: | %(path)s.info.clean
|
|
|
|
|
-.PHONY: %(path)s.info.clean
|
|
|
|
|
-%(path)s.info.clean:
|
|
|
|
|
- rm -f %(path)s*.gcda
|
|
|
|
|
-
|
|
|
|
|
-# accumulate coverage info
|
|
|
|
|
-.PHONY: %(path)s.info
|
|
|
|
|
-%(path)s.info:
|
|
|
|
|
- $(strip $(LCOV) -c \\
|
|
|
|
|
- $(addprefix -d ,$(wildcard %(path)s*.gcda)) \\
|
|
|
|
|
- --rc 'geninfo_adjust_src_path=$(shell pwd)' \\
|
|
|
|
|
- -o $@)
|
|
|
|
|
- $(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@
|
|
|
|
|
-ifdef COVERAGETARGET
|
|
|
|
|
- $(strip $(LCOV) -a $@ \\
|
|
|
|
|
- $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\
|
|
|
|
|
- -o $(COVERAGETARGET))
|
|
|
|
|
-endif
|
|
|
|
|
-"""
|
|
|
|
|
-GLOBALS = """
|
|
|
|
|
-//////////////// AUTOGENERATED TEST ////////////////
|
|
|
|
|
-#include "lfs.h"
|
|
|
|
|
|
|
+SUITE_PROLOGUE = """
|
|
|
|
|
+#include "runners/test_runner.h"
|
|
|
#include "bd/lfs_testbd.h"
|
|
#include "bd/lfs_testbd.h"
|
|
|
#include <stdio.h>
|
|
#include <stdio.h>
|
|
|
-extern const char *lfs_testbd_path;
|
|
|
|
|
-extern uint32_t lfs_testbd_cycles;
|
|
|
|
|
"""
|
|
"""
|
|
|
-DEFINES = {
|
|
|
|
|
- 'LFS_READ_SIZE': 16,
|
|
|
|
|
- 'LFS_PROG_SIZE': 'LFS_READ_SIZE',
|
|
|
|
|
- 'LFS_BLOCK_SIZE': 512,
|
|
|
|
|
- 'LFS_BLOCK_COUNT': 1024,
|
|
|
|
|
- 'LFS_BLOCK_CYCLES': -1,
|
|
|
|
|
- 'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)',
|
|
|
|
|
- 'LFS_LOOKAHEAD_SIZE': 16,
|
|
|
|
|
- 'LFS_ERASE_VALUE': 0xff,
|
|
|
|
|
- 'LFS_ERASE_CYCLES': 0,
|
|
|
|
|
- 'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_PROGERROR',
|
|
|
|
|
-}
|
|
|
|
|
-PROLOGUE = """
|
|
|
|
|
- // prologue
|
|
|
|
|
- __attribute__((unused)) lfs_t lfs;
|
|
|
|
|
- __attribute__((unused)) lfs_testbd_t bd;
|
|
|
|
|
- __attribute__((unused)) lfs_file_t file;
|
|
|
|
|
- __attribute__((unused)) lfs_dir_t dir;
|
|
|
|
|
- __attribute__((unused)) struct lfs_info info;
|
|
|
|
|
- __attribute__((unused)) char path[1024];
|
|
|
|
|
- __attribute__((unused)) uint8_t buffer[1024];
|
|
|
|
|
- __attribute__((unused)) lfs_size_t size;
|
|
|
|
|
- __attribute__((unused)) int err;
|
|
|
|
|
-
|
|
|
|
|
- __attribute__((unused)) const struct lfs_config cfg = {
|
|
|
|
|
- .context = &bd,
|
|
|
|
|
- .read = lfs_testbd_read,
|
|
|
|
|
- .prog = lfs_testbd_prog,
|
|
|
|
|
- .erase = lfs_testbd_erase,
|
|
|
|
|
- .sync = lfs_testbd_sync,
|
|
|
|
|
- .read_size = LFS_READ_SIZE,
|
|
|
|
|
- .prog_size = LFS_PROG_SIZE,
|
|
|
|
|
- .block_size = LFS_BLOCK_SIZE,
|
|
|
|
|
- .block_count = LFS_BLOCK_COUNT,
|
|
|
|
|
- .block_cycles = LFS_BLOCK_CYCLES,
|
|
|
|
|
- .cache_size = LFS_CACHE_SIZE,
|
|
|
|
|
- .lookahead_size = LFS_LOOKAHEAD_SIZE,
|
|
|
|
|
- };
|
|
|
|
|
-
|
|
|
|
|
- __attribute__((unused)) const struct lfs_testbd_config bdcfg = {
|
|
|
|
|
- .erase_value = LFS_ERASE_VALUE,
|
|
|
|
|
- .erase_cycles = LFS_ERASE_CYCLES,
|
|
|
|
|
- .badblock_behavior = LFS_BADBLOCK_BEHAVIOR,
|
|
|
|
|
- .power_cycles = lfs_testbd_cycles,
|
|
|
|
|
- };
|
|
|
|
|
-
|
|
|
|
|
- lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0;
|
|
|
|
|
|
|
+CASE_PROLOGUE = """
|
|
|
"""
|
|
"""
|
|
|
-EPILOGUE = """
|
|
|
|
|
- // epilogue
|
|
|
|
|
- lfs_testbd_destroy(&cfg) => 0;
|
|
|
|
|
|
|
+CASE_EPILOGUE = """
|
|
|
"""
|
|
"""
|
|
|
-PASS = '\033[32m✓\033[0m'
|
|
|
|
|
-FAIL = '\033[31m✗\033[0m'
|
|
|
|
|
|
|
|
|
|
-class TestFailure(Exception):
|
|
|
|
|
- def __init__(self, case, returncode=None, stdout=None, assert_=None):
|
|
|
|
|
- self.case = case
|
|
|
|
|
- self.returncode = returncode
|
|
|
|
|
- self.stdout = stdout
|
|
|
|
|
- self.assert_ = assert_
|
|
|
|
|
|
|
|
|
|
-class TestCase:
|
|
|
|
|
- def __init__(self, config, filter=filter,
|
|
|
|
|
- suite=None, caseno=None, lineno=None, **_):
|
|
|
|
|
- self.config = config
|
|
|
|
|
- self.filter = filter
|
|
|
|
|
- self.suite = suite
|
|
|
|
|
- self.caseno = caseno
|
|
|
|
|
- self.lineno = lineno
|
|
|
|
|
-
|
|
|
|
|
- self.code = config['code']
|
|
|
|
|
- self.code_lineno = config['code_lineno']
|
|
|
|
|
- self.defines = config.get('define', {})
|
|
|
|
|
- self.if_ = config.get('if', None)
|
|
|
|
|
- self.in_ = config.get('in', None)
|
|
|
|
|
-
|
|
|
|
|
- self.result = None
|
|
|
|
|
-
|
|
|
|
|
- def __str__(self):
|
|
|
|
|
- if hasattr(self, 'permno'):
|
|
|
|
|
- if any(k not in self.case.defines for k in self.defines):
|
|
|
|
|
- return '%s#%d#%d (%s)' % (
|
|
|
|
|
- self.suite.name, self.caseno, self.permno, ', '.join(
|
|
|
|
|
- '%s=%s' % (k, v) for k, v in self.defines.items()
|
|
|
|
|
- if k not in self.case.defines))
|
|
|
|
|
- else:
|
|
|
|
|
- return '%s#%d#%d' % (
|
|
|
|
|
- self.suite.name, self.caseno, self.permno)
|
|
|
|
|
- else:
|
|
|
|
|
- return '%s#%d' % (
|
|
|
|
|
- self.suite.name, self.caseno)
|
|
|
|
|
-
|
|
|
|
|
- def permute(self, class_=None, defines={}, permno=None, **_):
|
|
|
|
|
- ncase = (class_ or type(self))(self.config)
|
|
|
|
|
- for k, v in self.__dict__.items():
|
|
|
|
|
- setattr(ncase, k, v)
|
|
|
|
|
- ncase.case = self
|
|
|
|
|
- ncase.perms = [ncase]
|
|
|
|
|
- ncase.permno = permno
|
|
|
|
|
- ncase.defines = defines
|
|
|
|
|
- return ncase
|
|
|
|
|
-
|
|
|
|
|
- def build(self, f, **_):
|
|
|
|
|
- # prologue
|
|
|
|
|
- for k, v in sorted(self.defines.items()):
|
|
|
|
|
- if k not in self.suite.defines:
|
|
|
|
|
- f.write('#define %s %s\n' % (k, v))
|
|
|
|
|
-
|
|
|
|
|
- f.write('void test_case%d(%s) {' % (self.caseno, ','.join(
|
|
|
|
|
- '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k
|
|
|
|
|
- for k in sorted(self.perms[0].defines)
|
|
|
|
|
- if k not in self.defines)))
|
|
|
|
|
-
|
|
|
|
|
- f.write(PROLOGUE)
|
|
|
|
|
- f.write('\n')
|
|
|
|
|
- f.write(4*' '+'// test case %d\n' % self.caseno)
|
|
|
|
|
- f.write(4*' '+'#line %d "%s"\n' % (self.code_lineno, self.suite.path))
|
|
|
|
|
-
|
|
|
|
|
- # test case goes here
|
|
|
|
|
- f.write(self.code)
|
|
|
|
|
-
|
|
|
|
|
- # epilogue
|
|
|
|
|
- f.write(EPILOGUE)
|
|
|
|
|
- f.write('}\n')
|
|
|
|
|
-
|
|
|
|
|
- for k, v in sorted(self.defines.items()):
|
|
|
|
|
- if k not in self.suite.defines:
|
|
|
|
|
- f.write('#undef %s\n' % k)
|
|
|
|
|
-
|
|
|
|
|
- def shouldtest(self, **args):
|
|
|
|
|
- if (self.filter is not None and
|
|
|
|
|
- len(self.filter) >= 1 and
|
|
|
|
|
- self.filter[0] != self.caseno):
|
|
|
|
|
- return False
|
|
|
|
|
- elif (self.filter is not None and
|
|
|
|
|
- len(self.filter) >= 2 and
|
|
|
|
|
- self.filter[1] != self.permno):
|
|
|
|
|
- return False
|
|
|
|
|
- elif args.get('no_internal') and self.in_ is not None:
|
|
|
|
|
- return False
|
|
|
|
|
- elif self.if_ is not None:
|
|
|
|
|
- if_ = self.if_
|
|
|
|
|
- while True:
|
|
|
|
|
- for k, v in sorted(self.defines.items(),
|
|
|
|
|
- key=lambda x: len(x[0]), reverse=True):
|
|
|
|
|
- if k in if_:
|
|
|
|
|
- if_ = if_.replace(k, '(%s)' % v)
|
|
|
|
|
- break
|
|
|
|
|
- else:
|
|
|
|
|
- break
|
|
|
|
|
- if_ = (
|
|
|
|
|
- re.sub('(\&\&|\?)', ' and ',
|
|
|
|
|
- re.sub('(\|\||:)', ' or ',
|
|
|
|
|
- re.sub('!(?!=)', ' not ', if_))))
|
|
|
|
|
- return eval(if_)
|
|
|
|
|
- else:
|
|
|
|
|
- return True
|
|
|
|
|
-
|
|
|
|
|
- def test(self, exec=[], persist=False, cycles=None,
|
|
|
|
|
- gdb=False, failure=None, disk=None, **args):
|
|
|
|
|
- # build command
|
|
|
|
|
- cmd = exec + ['./%s.test' % self.suite.path,
|
|
|
|
|
- repr(self.caseno), repr(self.permno)]
|
|
|
|
|
-
|
|
|
|
|
- # persist disk or keep in RAM for speed?
|
|
|
|
|
- if persist:
|
|
|
|
|
- if not disk:
|
|
|
|
|
- disk = self.suite.path + '.disk'
|
|
|
|
|
- if persist != 'noerase':
|
|
|
|
|
- try:
|
|
|
|
|
- with open(disk, 'w') as f:
|
|
|
|
|
- f.truncate(0)
|
|
|
|
|
- if args.get('verbose'):
|
|
|
|
|
- print('truncate --size=0', disk)
|
|
|
|
|
- except FileNotFoundError:
|
|
|
|
|
- pass
|
|
|
|
|
-
|
|
|
|
|
- cmd.append(disk)
|
|
|
|
|
-
|
|
|
|
|
- # simulate power-loss after n cycles?
|
|
|
|
|
- if cycles:
|
|
|
|
|
- cmd.append(str(cycles))
|
|
|
|
|
-
|
|
|
|
|
- # failed? drop into debugger?
|
|
|
|
|
- if gdb and failure:
|
|
|
|
|
- ncmd = ['gdb']
|
|
|
|
|
- if gdb == 'assert':
|
|
|
|
|
- ncmd.extend(['-ex', 'r'])
|
|
|
|
|
- if failure.assert_:
|
|
|
|
|
- ncmd.extend(['-ex', 'up 2'])
|
|
|
|
|
- elif gdb == 'main':
|
|
|
|
|
- ncmd.extend([
|
|
|
|
|
- '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno),
|
|
|
|
|
- '-ex', 'r'])
|
|
|
|
|
- ncmd.extend(['--args'] + cmd)
|
|
|
|
|
-
|
|
|
|
|
- if args.get('verbose'):
|
|
|
|
|
- print(' '.join(shlex.quote(c) for c in ncmd))
|
|
|
|
|
- signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
|
|
|
- sys.exit(sp.call(ncmd))
|
|
|
|
|
-
|
|
|
|
|
- # run test case!
|
|
|
|
|
- mpty, spty = pty.openpty()
|
|
|
|
|
- if args.get('verbose'):
|
|
|
|
|
- print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
|
|
- proc = sp.Popen(cmd, stdout=spty, stderr=spty)
|
|
|
|
|
- os.close(spty)
|
|
|
|
|
- mpty = os.fdopen(mpty, 'r', 1)
|
|
|
|
|
- stdout = []
|
|
|
|
|
- assert_ = None
|
|
|
|
|
- try:
|
|
|
|
|
- while True:
|
|
|
|
|
- try:
|
|
|
|
|
- line = mpty.readline()
|
|
|
|
|
- except OSError as e:
|
|
|
|
|
- if e.errno == errno.EIO:
|
|
|
|
|
- break
|
|
|
|
|
- raise
|
|
|
|
|
- if not line:
|
|
|
|
|
- break;
|
|
|
|
|
- stdout.append(line)
|
|
|
|
|
- if args.get('verbose'):
|
|
|
|
|
- sys.stdout.write(line)
|
|
|
|
|
- # intercept asserts
|
|
|
|
|
- m = re.match(
|
|
|
|
|
- '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'
|
|
|
|
|
- .format('(?:\033\[[\d;]*.| )*', 'assert'),
|
|
|
|
|
- line)
|
|
|
|
|
- if m and assert_ is None:
|
|
|
|
|
- try:
|
|
|
|
|
- with open(m.group(1)) as f:
|
|
|
|
|
- lineno = int(m.group(2))
|
|
|
|
|
- line = (next(it.islice(f, lineno-1, None))
|
|
|
|
|
- .strip('\n'))
|
|
|
|
|
- assert_ = {
|
|
|
|
|
- 'path': m.group(1),
|
|
|
|
|
- 'line': line,
|
|
|
|
|
- 'lineno': lineno,
|
|
|
|
|
- 'message': m.group(3)}
|
|
|
|
|
- except:
|
|
|
|
|
- pass
|
|
|
|
|
- except KeyboardInterrupt:
|
|
|
|
|
- raise TestFailure(self, 1, stdout, None)
|
|
|
|
|
- proc.wait()
|
|
|
|
|
-
|
|
|
|
|
- # did we pass?
|
|
|
|
|
- if proc.returncode != 0:
|
|
|
|
|
- raise TestFailure(self, proc.returncode, stdout, assert_)
|
|
|
|
|
- else:
|
|
|
|
|
- return PASS
|
|
|
|
|
|
|
+def testpath(path):
|
|
|
|
|
+ path, *_ = path.split('#', 1)
|
|
|
|
|
+ return path
|
|
|
|
|
|
|
|
-class ValgrindTestCase(TestCase):
|
|
|
|
|
- def __init__(self, config, **args):
|
|
|
|
|
- self.leaky = config.get('leaky', False)
|
|
|
|
|
- super().__init__(config, **args)
|
|
|
|
|
|
|
+def testsuite(path):
|
|
|
|
|
+ suite = testpath(path)
|
|
|
|
|
+ suite = os.path.basename(suite)
|
|
|
|
|
+ if suite.endswith('.toml'):
|
|
|
|
|
+ suite = suite[:-len('.toml')]
|
|
|
|
|
+ return suite
|
|
|
|
|
|
|
|
- def shouldtest(self, **args):
|
|
|
|
|
- return not self.leaky and super().shouldtest(**args)
|
|
|
|
|
|
|
+def testcase(path):
|
|
|
|
|
+ _, case, *_ = path.split('#', 2)
|
|
|
|
|
+ return '%s#%s' % (testsuite(path), case)
|
|
|
|
|
|
|
|
- def test(self, exec=[], **args):
|
|
|
|
|
- verbose = args.get('verbose')
|
|
|
|
|
- uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1)
|
|
|
|
|
- exec = [
|
|
|
|
|
- 'valgrind',
|
|
|
|
|
- '--leak-check=full',
|
|
|
|
|
- ] + (['--undef-value-errors=no'] if uninit else []) + [
|
|
|
|
|
- ] + (['--track-origins=yes'] if not uninit else []) + [
|
|
|
|
|
- '--error-exitcode=4',
|
|
|
|
|
- '--error-limit=no',
|
|
|
|
|
- ] + (['--num-callers=1'] if not verbose else []) + [
|
|
|
|
|
- '-q'] + exec
|
|
|
|
|
- return super().test(exec=exec, **args)
|
|
|
|
|
-
|
|
|
|
|
-class ReentrantTestCase(TestCase):
|
|
|
|
|
- def __init__(self, config, **args):
|
|
|
|
|
- self.reentrant = config.get('reentrant', False)
|
|
|
|
|
- super().__init__(config, **args)
|
|
|
|
|
-
|
|
|
|
|
- def shouldtest(self, **args):
|
|
|
|
|
- return self.reentrant and super().shouldtest(**args)
|
|
|
|
|
-
|
|
|
|
|
- def test(self, persist=False, gdb=False, failure=None, **args):
|
|
|
|
|
- for cycles in it.count(1):
|
|
|
|
|
- # clear disk first?
|
|
|
|
|
- if cycles == 1 and persist != 'noerase':
|
|
|
|
|
- persist = 'erase'
|
|
|
|
|
- else:
|
|
|
|
|
- persist = 'noerase'
|
|
|
|
|
|
|
+# TODO move this out in other files
|
|
|
|
|
+def openio(path, mode='r'):
|
|
|
|
|
+ if path == '-':
|
|
|
|
|
+ if 'r' in mode:
|
|
|
|
|
+ return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
|
|
|
|
|
+ else:
|
|
|
|
|
+ return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
|
|
|
|
|
+ else:
|
|
|
|
|
+ return open(path, mode)
|
|
|
|
|
|
|
|
- # exact cycle we should drop into debugger?
|
|
|
|
|
- if gdb and failure and failure.cycleno == cycles:
|
|
|
|
|
- return super().test(gdb=gdb, persist=persist, cycles=cycles,
|
|
|
|
|
- failure=failure, **args)
|
|
|
|
|
|
|
+class TestCase:
|
|
|
|
|
+ # create a TestCase object from a config
|
|
|
|
|
+ def __init__(self, config, args={}):
|
|
|
|
|
+ self.name = config.pop('name')
|
|
|
|
|
+ self.path = config.pop('path')
|
|
|
|
|
+ self.suite = config.pop('suite')
|
|
|
|
|
+ self.lineno = config.pop('lineno', None)
|
|
|
|
|
+ self.if_ = config.pop('if', None)
|
|
|
|
|
+ if isinstance(self.if_, bool):
|
|
|
|
|
+ self.if_ = 'true' if self.if_ else 'false'
|
|
|
|
|
+ self.code = config.pop('code')
|
|
|
|
|
+ self.code_lineno = config.pop('code_lineno', None)
|
|
|
|
|
+ self.in_ = config.pop('in',
|
|
|
|
|
+ config.pop('suite_in', None))
|
|
|
|
|
+
|
|
|
|
|
+ self.normal = config.pop('normal',
|
|
|
|
|
+ config.pop('suite_normal', True))
|
|
|
|
|
+ self.reentrant = config.pop('reentrant',
|
|
|
|
|
+ config.pop('suite_reentrant', False))
|
|
|
|
|
+
|
|
|
|
|
+ # figure out defines and build possible permutations
|
|
|
|
|
+ self.defines = set()
|
|
|
|
|
+ self.permutations = []
|
|
|
|
|
+
|
|
|
|
|
+ suite_defines = config.pop('suite_defines', {})
|
|
|
|
|
+ if not isinstance(suite_defines, list):
|
|
|
|
|
+ suite_defines = [suite_defines]
|
|
|
|
|
+ defines = config.pop('defines', {})
|
|
|
|
|
+ if not isinstance(defines, list):
|
|
|
|
|
+ defines = [defines]
|
|
|
|
|
+
|
|
|
|
|
+ # build possible permutations
|
|
|
|
|
+ for suite_defines_ in suite_defines:
|
|
|
|
|
+ self.defines |= suite_defines_.keys()
|
|
|
|
|
+ for defines_ in defines:
|
|
|
|
|
+ self.defines |= defines_.keys()
|
|
|
|
|
+ self.permutations.extend(map(dict, it.product(*(
|
|
|
|
|
+ [(k, v) for v in (vs if isinstance(vs, list) else [vs])]
|
|
|
|
|
+ for k, vs in sorted(
|
|
|
|
|
+ (suite_defines_ | defines_).items())))))
|
|
|
|
|
+
|
|
|
|
|
+ for k in config.keys():
|
|
|
|
|
+ print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
|
|
|
|
|
+ % (self.id(), k),
|
|
|
|
|
+ file=sys.stderr)
|
|
|
|
|
+
|
|
|
|
|
+ def id(self):
|
|
|
|
|
+ return '%s#%s' % (self.suite, self.name)
|
|
|
|
|
|
|
|
- # run tests, but kill the program after prog/erase has
|
|
|
|
|
- # been hit n cycles. We exit with a special return code if the
|
|
|
|
|
- # program has not finished, since this isn't a test failure.
|
|
|
|
|
- try:
|
|
|
|
|
- return super().test(persist=persist, cycles=cycles, **args)
|
|
|
|
|
- except TestFailure as nfailure:
|
|
|
|
|
- if nfailure.returncode == 33:
|
|
|
|
|
- continue
|
|
|
|
|
- else:
|
|
|
|
|
- nfailure.cycleno = cycles
|
|
|
|
|
- raise
|
|
|
|
|
|
|
|
|
|
class TestSuite:
|
|
class TestSuite:
|
|
|
- def __init__(self, path, classes=[TestCase], defines={},
|
|
|
|
|
- filter=None, **args):
|
|
|
|
|
- self.name = os.path.basename(path)
|
|
|
|
|
- if self.name.endswith('.toml'):
|
|
|
|
|
- self.name = self.name[:-len('.toml')]
|
|
|
|
|
- if args.get('build_dir'):
|
|
|
|
|
- self.toml = path
|
|
|
|
|
- self.path = args['build_dir'] + '/' + path
|
|
|
|
|
- else:
|
|
|
|
|
- self.toml = path
|
|
|
|
|
- self.path = path
|
|
|
|
|
- self.classes = classes
|
|
|
|
|
- self.defines = defines.copy()
|
|
|
|
|
- self.filter = filter
|
|
|
|
|
|
|
+ # create a TestSuite object from a toml file
|
|
|
|
|
+ def __init__(self, path, args={}):
|
|
|
|
|
+ self.name = testsuite(path)
|
|
|
|
|
+ self.path = testpath(path)
|
|
|
|
|
|
|
|
- with open(self.toml) as f:
|
|
|
|
|
|
|
+ # load toml file and parse test cases
|
|
|
|
|
+ with open(self.path) as f:
|
|
|
# load tests
|
|
# load tests
|
|
|
config = toml.load(f)
|
|
config = toml.load(f)
|
|
|
|
|
|
|
|
# find line numbers
|
|
# find line numbers
|
|
|
f.seek(0)
|
|
f.seek(0)
|
|
|
- linenos = []
|
|
|
|
|
|
|
+ case_linenos = []
|
|
|
code_linenos = []
|
|
code_linenos = []
|
|
|
for i, line in enumerate(f):
|
|
for i, line in enumerate(f):
|
|
|
- if re.match(r'\[\[\s*case\s*\]\]', line):
|
|
|
|
|
- linenos.append(i+1)
|
|
|
|
|
- if re.match(r'code\s*=\s*(\'\'\'|""")', line):
|
|
|
|
|
|
|
+ match = re.match(
|
|
|
|
|
+ '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
|
|
|
|
|
+ '|' '(?P<code>code\s*=)',
|
|
|
|
|
+ line)
|
|
|
|
|
+ if match and match.group('case'):
|
|
|
|
|
+ case_linenos.append((i+1, match.group('name')))
|
|
|
|
|
+ elif match and match.group('code'):
|
|
|
code_linenos.append(i+2)
|
|
code_linenos.append(i+2)
|
|
|
|
|
|
|
|
- code_linenos.reverse()
|
|
|
|
|
-
|
|
|
|
|
- # grab global config
|
|
|
|
|
- for k, v in config.get('define', {}).items():
|
|
|
|
|
- if k not in self.defines:
|
|
|
|
|
- self.defines[k] = v
|
|
|
|
|
- self.code = config.get('code', None)
|
|
|
|
|
- if self.code is not None:
|
|
|
|
|
- self.code_lineno = code_linenos.pop()
|
|
|
|
|
-
|
|
|
|
|
- # create initial test cases
|
|
|
|
|
- self.cases = []
|
|
|
|
|
- for i, (case, lineno) in enumerate(zip(config['case'], linenos)):
|
|
|
|
|
- # code lineno?
|
|
|
|
|
- if 'code' in case:
|
|
|
|
|
- case['code_lineno'] = code_linenos.pop()
|
|
|
|
|
- # merge conditions if necessary
|
|
|
|
|
- if 'if' in config and 'if' in case:
|
|
|
|
|
- case['if'] = '(%s) && (%s)' % (config['if'], case['if'])
|
|
|
|
|
- elif 'if' in config:
|
|
|
|
|
- case['if'] = config['if']
|
|
|
|
|
- # initialize test case
|
|
|
|
|
- self.cases.append(TestCase(case, filter=filter,
|
|
|
|
|
- suite=self, caseno=i+1, lineno=lineno, **args))
|
|
|
|
|
-
|
|
|
|
|
- def __str__(self):
|
|
|
|
|
|
|
+ # sort in case toml parsing did not retain order
|
|
|
|
|
+ case_linenos.sort()
|
|
|
|
|
+
|
|
|
|
|
+ cases = config.pop('cases')
|
|
|
|
|
+ for (lineno, name), (nlineno, _) in it.zip_longest(
|
|
|
|
|
+ case_linenos, case_linenos[1:],
|
|
|
|
|
+ fillvalue=(float('inf'), None)):
|
|
|
|
|
+ code_lineno = min(
|
|
|
|
|
+ (l for l in code_linenos if l >= lineno and l < nlineno),
|
|
|
|
|
+ default=None)
|
|
|
|
|
+ cases[name]['lineno'] = lineno
|
|
|
|
|
+ cases[name]['code_lineno'] = code_lineno
|
|
|
|
|
+
|
|
|
|
|
+ self.if_ = config.pop('if', None)
|
|
|
|
|
+ if isinstance(self.if_, bool):
|
|
|
|
|
+ self.if_ = 'true' if self.if_ else 'false'
|
|
|
|
|
+
|
|
|
|
|
+ self.code = config.pop('code', None)
|
|
|
|
|
+ self.code_lineno = min(
|
|
|
|
|
+ (l for l in code_linenos
|
|
|
|
|
+ if not case_linenos or l < case_linenos[0][0]),
|
|
|
|
|
+ default=None)
|
|
|
|
|
+
|
|
|
|
|
+ # a couple of these we just forward to all cases
|
|
|
|
|
+ defines = config.pop('defines', {})
|
|
|
|
|
+ in_ = config.pop('in', None)
|
|
|
|
|
+ normal = config.pop('normal', True)
|
|
|
|
|
+ reentrant = config.pop('reentrant', False)
|
|
|
|
|
+
|
|
|
|
|
+ self.cases = []
|
|
|
|
|
+ for name, case in sorted(cases.items(),
|
|
|
|
|
+ key=lambda c: c[1].get('lineno')):
|
|
|
|
|
+ self.cases.append(TestCase(config={
|
|
|
|
|
+ 'name': name,
|
|
|
|
|
+ 'path': path + (':%d' % case['lineno']
|
|
|
|
|
+ if 'lineno' in case else ''),
|
|
|
|
|
+ 'suite': self.name,
|
|
|
|
|
+ 'suite_defines': defines,
|
|
|
|
|
+ 'suite_in': in_,
|
|
|
|
|
+ 'suite_normal': normal,
|
|
|
|
|
+ 'suite_reentrant': reentrant,
|
|
|
|
|
+ **case}))
|
|
|
|
|
+
|
|
|
|
|
+ # combine per-case defines
|
|
|
|
|
+ self.defines = set.union(*(
|
|
|
|
|
+ set(case.defines) for case in self.cases))
|
|
|
|
|
+
|
|
|
|
|
+ # combine other per-case things
|
|
|
|
|
+ self.normal = any(case.normal for case in self.cases)
|
|
|
|
|
+ self.reentrant = any(case.reentrant for case in self.cases)
|
|
|
|
|
+
|
|
|
|
|
+ for k in config.keys():
|
|
|
|
|
+ print('\x1b[01;33mwarning:\x1b[m in %s, found unused key %r'
|
|
|
|
|
+ % (self.id(), k),
|
|
|
|
|
+ file=sys.stderr)
|
|
|
|
|
+
|
|
|
|
|
+ def id(self):
|
|
|
return self.name
|
|
return self.name
|
|
|
|
|
|
|
|
- def __lt__(self, other):
|
|
|
|
|
- return self.name < other.name
|
|
|
|
|
-
|
|
|
|
|
- def permute(self, **args):
|
|
|
|
|
- for case in self.cases:
|
|
|
|
|
- # lets find all parameterized definitions, in one of [args.D,
|
|
|
|
|
- # suite.defines, case.defines, DEFINES]. Note that each of these
|
|
|
|
|
- # can be either a dict of defines, or a list of dicts, expressing
|
|
|
|
|
- # an initial set of permutations.
|
|
|
|
|
- pending = [{}]
|
|
|
|
|
- for inits in [self.defines, case.defines, DEFINES]:
|
|
|
|
|
- if not isinstance(inits, list):
|
|
|
|
|
- inits = [inits]
|
|
|
|
|
-
|
|
|
|
|
- npending = []
|
|
|
|
|
- for init, pinit in it.product(inits, pending):
|
|
|
|
|
- ninit = pinit.copy()
|
|
|
|
|
- for k, v in init.items():
|
|
|
|
|
- if k not in ninit:
|
|
|
|
|
- try:
|
|
|
|
|
- ninit[k] = eval(v)
|
|
|
|
|
- except:
|
|
|
|
|
- ninit[k] = v
|
|
|
|
|
- npending.append(ninit)
|
|
|
|
|
-
|
|
|
|
|
- pending = npending
|
|
|
|
|
-
|
|
|
|
|
- # expand permutations
|
|
|
|
|
- pending = list(reversed(pending))
|
|
|
|
|
- expanded = []
|
|
|
|
|
- while pending:
|
|
|
|
|
- perm = pending.pop()
|
|
|
|
|
- for k, v in sorted(perm.items()):
|
|
|
|
|
- if not isinstance(v, str) and isinstance(v, abc.Iterable):
|
|
|
|
|
- for nv in reversed(v):
|
|
|
|
|
- nperm = perm.copy()
|
|
|
|
|
- nperm[k] = nv
|
|
|
|
|
- pending.append(nperm)
|
|
|
|
|
- break
|
|
|
|
|
- else:
|
|
|
|
|
- expanded.append(perm)
|
|
|
|
|
-
|
|
|
|
|
- # generate permutations
|
|
|
|
|
- case.perms = []
|
|
|
|
|
- for i, (class_, defines) in enumerate(
|
|
|
|
|
- it.product(self.classes, expanded)):
|
|
|
|
|
- case.perms.append(case.permute(
|
|
|
|
|
- class_, defines, permno=i+1, **args))
|
|
|
|
|
-
|
|
|
|
|
- # also track non-unique defines
|
|
|
|
|
- case.defines = {}
|
|
|
|
|
- for k, v in case.perms[0].defines.items():
|
|
|
|
|
- if all(perm.defines[k] == v for perm in case.perms):
|
|
|
|
|
- case.defines[k] = v
|
|
|
|
|
-
|
|
|
|
|
- # track all perms and non-unique defines
|
|
|
|
|
- self.perms = []
|
|
|
|
|
- for case in self.cases:
|
|
|
|
|
- self.perms.extend(case.perms)
|
|
|
|
|
-
|
|
|
|
|
- self.defines = {}
|
|
|
|
|
- for k, v in self.perms[0].defines.items():
|
|
|
|
|
- if all(perm.defines.get(k, None) == v for perm in self.perms):
|
|
|
|
|
- self.defines[k] = v
|
|
|
|
|
-
|
|
|
|
|
- return self.perms
|
|
|
|
|
-
|
|
|
|
|
- def build(self, **args):
|
|
|
|
|
- # build test files
|
|
|
|
|
- tf = open(self.path + '.test.tc', 'w')
|
|
|
|
|
- tf.write(GLOBALS)
|
|
|
|
|
- if self.code is not None:
|
|
|
|
|
- tf.write('#line %d "%s"\n' % (self.code_lineno, self.path))
|
|
|
|
|
- tf.write(self.code)
|
|
|
|
|
-
|
|
|
|
|
- tfs = {None: tf}
|
|
|
|
|
- for case in self.cases:
|
|
|
|
|
- if case.in_ not in tfs:
|
|
|
|
|
- tfs[case.in_] = open(self.path+'.'+
|
|
|
|
|
- re.sub('(\.c)?$', '.tc', case.in_.replace('/', '.')), 'w')
|
|
|
|
|
- tfs[case.in_].write('#line 1 "%s"\n' % case.in_)
|
|
|
|
|
- with open(case.in_) as f:
|
|
|
|
|
- for line in f:
|
|
|
|
|
- tfs[case.in_].write(line)
|
|
|
|
|
- tfs[case.in_].write('\n')
|
|
|
|
|
- tfs[case.in_].write(GLOBALS)
|
|
|
|
|
-
|
|
|
|
|
- tfs[case.in_].write('\n')
|
|
|
|
|
- case.build(tfs[case.in_], **args)
|
|
|
|
|
-
|
|
|
|
|
- tf.write('\n')
|
|
|
|
|
- tf.write('const char *lfs_testbd_path;\n')
|
|
|
|
|
- tf.write('uint32_t lfs_testbd_cycles;\n')
|
|
|
|
|
- tf.write('int main(int argc, char **argv) {\n')
|
|
|
|
|
- tf.write(4*' '+'int case_ = (argc > 1) ? atoi(argv[1]) : 0;\n')
|
|
|
|
|
- tf.write(4*' '+'int perm = (argc > 2) ? atoi(argv[2]) : 0;\n')
|
|
|
|
|
- tf.write(4*' '+'lfs_testbd_path = (argc > 3) ? argv[3] : NULL;\n')
|
|
|
|
|
- tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n')
|
|
|
|
|
- for perm in self.perms:
|
|
|
|
|
- # test declaration
|
|
|
|
|
- tf.write(4*' '+'extern void test_case%d(%s);\n' % (
|
|
|
|
|
- perm.caseno, ', '.join(
|
|
|
|
|
- 'intmax_t %s' % k for k in sorted(perm.defines)
|
|
|
|
|
- if k not in perm.case.defines)))
|
|
|
|
|
- # test call
|
|
|
|
|
- tf.write(4*' '+
|
|
|
|
|
- 'if (argc < 3 || (case_ == %d && perm == %d)) {'
|
|
|
|
|
- ' test_case%d(%s); '
|
|
|
|
|
- '}\n' % (perm.caseno, perm.permno, perm.caseno, ', '.join(
|
|
|
|
|
- str(v) for k, v in sorted(perm.defines.items())
|
|
|
|
|
- if k not in perm.case.defines)))
|
|
|
|
|
- tf.write('}\n')
|
|
|
|
|
-
|
|
|
|
|
- for tf in tfs.values():
|
|
|
|
|
- tf.close()
|
|
|
|
|
-
|
|
|
|
|
- # write makefiles
|
|
|
|
|
- with open(self.path + '.mk', 'w') as mk:
|
|
|
|
|
- mk.write(RULES.replace(4*' ', '\t') % dict(path=self.path))
|
|
|
|
|
- mk.write('\n')
|
|
|
|
|
-
|
|
|
|
|
- # add coverage hooks?
|
|
|
|
|
- if args.get('coverage'):
|
|
|
|
|
- mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict(
|
|
|
|
|
- path=self.path))
|
|
|
|
|
- mk.write('\n')
|
|
|
|
|
-
|
|
|
|
|
- # add truly global defines globally
|
|
|
|
|
- for k, v in sorted(self.defines.items()):
|
|
|
|
|
- mk.write('%s.test: override CFLAGS += -D%s=%r\n'
|
|
|
|
|
- % (self.path, k, v))
|
|
|
|
|
-
|
|
|
|
|
- for path in tfs:
|
|
|
|
|
- if path is None:
|
|
|
|
|
- mk.write('%s: %s | %s\n' % (
|
|
|
|
|
- self.path+'.test.c',
|
|
|
|
|
- self.toml,
|
|
|
|
|
- self.path+'.test.tc'))
|
|
|
|
|
- else:
|
|
|
|
|
- mk.write('%s: %s %s | %s\n' % (
|
|
|
|
|
- self.path+'.'+path.replace('/', '.'),
|
|
|
|
|
- self.toml,
|
|
|
|
|
- path,
|
|
|
|
|
- self.path+'.'+re.sub('(\.c)?$', '.tc',
|
|
|
|
|
- path.replace('/', '.'))))
|
|
|
|
|
- mk.write('\t./scripts/explode_asserts.py $| -o $@\n')
|
|
|
|
|
-
|
|
|
|
|
- self.makefile = self.path + '.mk'
|
|
|
|
|
- self.target = self.path + '.test'
|
|
|
|
|
- return self.makefile, self.target
|
|
|
|
|
-
|
|
|
|
|
- def test(self, **args):
|
|
|
|
|
- # run test suite!
|
|
|
|
|
- if not args.get('verbose', True):
|
|
|
|
|
- sys.stdout.write(self.name + ' ')
|
|
|
|
|
- sys.stdout.flush()
|
|
|
|
|
- for perm in self.perms:
|
|
|
|
|
- if not perm.shouldtest(**args):
|
|
|
|
|
- continue
|
|
|
|
|
|
|
|
|
|
- try:
|
|
|
|
|
- result = perm.test(**args)
|
|
|
|
|
- except TestFailure as failure:
|
|
|
|
|
- perm.result = failure
|
|
|
|
|
- if not args.get('verbose', True):
|
|
|
|
|
- sys.stdout.write(FAIL)
|
|
|
|
|
- sys.stdout.flush()
|
|
|
|
|
- if not args.get('keep_going'):
|
|
|
|
|
- if not args.get('verbose', True):
|
|
|
|
|
- sys.stdout.write('\n')
|
|
|
|
|
- raise
|
|
|
|
|
- else:
|
|
|
|
|
- perm.result = PASS
|
|
|
|
|
- if not args.get('verbose', True):
|
|
|
|
|
- sys.stdout.write(PASS)
|
|
|
|
|
- sys.stdout.flush()
|
|
|
|
|
|
|
|
|
|
- if not args.get('verbose', True):
|
|
|
|
|
- sys.stdout.write('\n')
|
|
|
|
|
|
|
+def compile(**args):
|
|
|
|
|
+ # find .toml files
|
|
|
|
|
+ paths = []
|
|
|
|
|
+ for path in args.get('test_ids', TEST_PATHS):
|
|
|
|
|
+ if os.path.isdir(path):
|
|
|
|
|
+ path = path + '/*.toml'
|
|
|
|
|
|
|
|
-def main(**args):
|
|
|
|
|
- # figure out explicit defines
|
|
|
|
|
- defines = {}
|
|
|
|
|
- for define in args['D']:
|
|
|
|
|
- k, v, *_ = define.split('=', 2) + ['']
|
|
|
|
|
- defines[k] = v
|
|
|
|
|
-
|
|
|
|
|
- # and what class of TestCase to run
|
|
|
|
|
- classes = []
|
|
|
|
|
- if args.get('normal'):
|
|
|
|
|
- classes.append(TestCase)
|
|
|
|
|
- if args.get('reentrant'):
|
|
|
|
|
- classes.append(ReentrantTestCase)
|
|
|
|
|
|
|
+ for path in glob.glob(path):
|
|
|
|
|
+ paths.append(path)
|
|
|
|
|
+
|
|
|
|
|
+ if not paths:
|
|
|
|
|
+ print('no test suites found in %r?' % args['test_ids'])
|
|
|
|
|
+ sys.exit(-1)
|
|
|
|
|
+
|
|
|
|
|
+ if not args.get('source'):
|
|
|
|
|
+ if len(paths) > 1:
|
|
|
|
|
+ print('more than one test suite for compilation? (%r)'
|
|
|
|
|
+ % args['test_ids'])
|
|
|
|
|
+ sys.exit(-1)
|
|
|
|
|
+
|
|
|
|
|
+ # load our suite
|
|
|
|
|
+ suite = TestSuite(paths[0])
|
|
|
|
|
+ else:
|
|
|
|
|
+ # load all suites
|
|
|
|
|
+ suites = [TestSuite(path) for path in paths]
|
|
|
|
|
+ suites.sort(key=lambda s: s.name)
|
|
|
|
|
+
|
|
|
|
|
+ # write generated test source
|
|
|
|
|
+ if 'output' in args:
|
|
|
|
|
+ with openio(args['output'], 'w') as f:
|
|
|
|
|
+ _write = f.write
|
|
|
|
|
+ def write(s):
|
|
|
|
|
+ f.lineno += s.count('\n')
|
|
|
|
|
+ _write(s)
|
|
|
|
|
+ def writeln(s=''):
|
|
|
|
|
+ f.lineno += s.count('\n') + 1
|
|
|
|
|
+ _write(s)
|
|
|
|
|
+ _write('\n')
|
|
|
|
|
+ f.lineno = 1
|
|
|
|
|
+ f.write = write
|
|
|
|
|
+ f.writeln = writeln
|
|
|
|
|
+
|
|
|
|
|
+ # redirect littlefs tracing
|
|
|
|
|
+ f.writeln('#define LFS_TRACE_(fmt, ...) do { \\')
|
|
|
|
|
+ f.writeln(8*' '+'extern FILE *test_trace; \\')
|
|
|
|
|
+ f.writeln(8*' '+'if (test_trace) { \\')
|
|
|
|
|
+ f.writeln(12*' '+'fprintf(test_trace, '
|
|
|
|
|
+ '"%s:%d:trace: " fmt "%s\\n", \\')
|
|
|
|
|
+ f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\')
|
|
|
|
|
+ f.writeln(8*' '+'} \\')
|
|
|
|
|
+ f.writeln(4*' '+'} while (0)')
|
|
|
|
|
+ f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")')
|
|
|
|
|
+ f.writeln('#define LFS_TESTBD_TRACE(...) '
|
|
|
|
|
+ 'LFS_TRACE_(__VA_ARGS__, "")')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # write out generated functions, this can end up in different
|
|
|
|
|
+ # files depending on the "in" attribute
|
|
|
|
|
+ #
|
|
|
|
|
+ # note it's up to the specific generated file to declare
|
|
|
|
|
+ # the test defines
|
|
|
|
|
+ def write_case_functions(f, suite, case):
|
|
|
|
|
+ # create case define functions
|
|
|
|
|
+ if case.defines:
|
|
|
|
|
+ # deduplicate defines by value to try to reduce the
|
|
|
|
|
+ # number of functions we generate
|
|
|
|
|
+ define_cbs = {}
|
|
|
|
|
+ for i, defines in enumerate(case.permutations):
|
|
|
|
|
+ for k, v in sorted(defines.items()):
|
|
|
|
|
+ if v not in define_cbs:
|
|
|
|
|
+ name = ('__test__%s__%s__%s__%d'
|
|
|
|
|
+ % (suite.name, case.name, k, i))
|
|
|
|
|
+ define_cbs[v] = name
|
|
|
|
|
+ f.writeln('intmax_t %s(void) {' % name)
|
|
|
|
|
+ f.writeln(4*' '+'return %s;' % v)
|
|
|
|
|
+ f.writeln('}')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+ f.writeln('intmax_t (*const *const '
|
|
|
|
|
+ '__test__%s__%s__defines[])(void) = {'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ for defines in case.permutations:
|
|
|
|
|
+ f.writeln(4*' '+'(intmax_t (*const[])(void)){')
|
|
|
|
|
+ for define in sorted(suite.defines):
|
|
|
|
|
+ f.writeln(8*' '+'%s,' % (
|
|
|
|
|
+ define_cbs[defines[define]]
|
|
|
|
|
+ if define in defines
|
|
|
|
|
+ else 'NULL'))
|
|
|
|
|
+ f.writeln(4*' '+'},')
|
|
|
|
|
+ f.writeln('};')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # create case filter function
|
|
|
|
|
+ if suite.if_ is not None or case.if_ is not None:
|
|
|
|
|
+ f.writeln('bool __test__%s__%s__filter(void) {'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln(4*' '+'return %s;'
|
|
|
|
|
+ % ' && '.join('(%s)' % if_
|
|
|
|
|
+ for if_ in [suite.if_, case.if_]
|
|
|
|
|
+ if if_ is not None))
|
|
|
|
|
+ f.writeln('}')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # create case run function
|
|
|
|
|
+ f.writeln('void __test__%s__%s__run('
|
|
|
|
|
+ '__attribute__((unused)) struct lfs_config *cfg) {'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ if CASE_PROLOGUE.strip():
|
|
|
|
|
+ f.writeln(4*' '+'%s'
|
|
|
|
|
+ % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+ f.writeln(4*' '+'// test case %s' % case.id())
|
|
|
|
|
+ if case.code_lineno is not None:
|
|
|
|
|
+ f.writeln(4*' '+'#line %d "%s"'
|
|
|
|
|
+ % (case.code_lineno, suite.path))
|
|
|
|
|
+ f.write(case.code)
|
|
|
|
|
+ if case.code_lineno is not None:
|
|
|
|
|
+ f.writeln(4*' '+'#line %d "%s"'
|
|
|
|
|
+ % (f.lineno+1, args['output']))
|
|
|
|
|
+ if CASE_EPILOGUE.strip():
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+ f.writeln(4*' '+'%s'
|
|
|
|
|
+ % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
|
|
|
|
|
+ f.writeln('}')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ if not args.get('source'):
|
|
|
|
|
+ # write test suite prologue
|
|
|
|
|
+ f.writeln('%s' % SUITE_PROLOGUE.strip())
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+ if suite.code is not None:
|
|
|
|
|
+ if suite.code_lineno is not None:
|
|
|
|
|
+ f.writeln('#line %d "%s"'
|
|
|
|
|
+ % (suite.code_lineno, suite.path))
|
|
|
|
|
+ f.write(suite.code)
|
|
|
|
|
+ if suite.code_lineno is not None:
|
|
|
|
|
+ f.writeln('#line %d "%s"'
|
|
|
|
|
+ % (f.lineno+1, args['output']))
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ if suite.defines:
|
|
|
|
|
+ for i, define in enumerate(sorted(suite.defines)):
|
|
|
|
|
+ f.writeln('#ifndef %s' % define)
|
|
|
|
|
+ f.writeln('#define %-24s test_define(%d)'
|
|
|
|
|
+ % (define, i))
|
|
|
|
|
+ f.writeln('#endif')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ for case in suite.cases:
|
|
|
|
|
+ # create case functions
|
|
|
|
|
+ if case.in_ is None:
|
|
|
|
|
+ write_case_functions(f, suite, case)
|
|
|
|
|
+ else:
|
|
|
|
|
+ if case.defines:
|
|
|
|
|
+ f.writeln('extern intmax_t (*const *const '
|
|
|
|
|
+ '__test__%s__%s__defines[])(void);'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ if suite.if_ is not None or case.if_ is not None:
|
|
|
|
|
+ f.writeln('extern bool __test__%s__%s__filter('
|
|
|
|
|
+ 'void);'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln('extern void __test__%s__%s__run('
|
|
|
|
|
+ 'struct lfs_config *cfg);'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # create case struct
|
|
|
|
|
+ f.writeln('const struct test_case __test__%s__%s__case = {'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln(4*' '+'.id = "%s",' % case.id())
|
|
|
|
|
+ f.writeln(4*' '+'.name = "%s",' % case.name)
|
|
|
|
|
+ f.writeln(4*' '+'.path = "%s",' % case.path)
|
|
|
|
|
+ f.writeln(4*' '+'.types = %s,'
|
|
|
|
|
+ % ' | '.join(filter(None, [
|
|
|
|
|
+ 'TEST_NORMAL' if case.normal else None,
|
|
|
|
|
+ 'TEST_REENTRANT' if case.reentrant else None])))
|
|
|
|
|
+ f.writeln(4*' '+'.permutations = %d,'
|
|
|
|
|
+ % len(case.permutations))
|
|
|
|
|
+ if case.defines:
|
|
|
|
|
+ f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ if suite.if_ is not None or case.if_ is not None:
|
|
|
|
|
+ f.writeln(4*' '+'.filter = __test__%s__%s__filter,'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln(4*' '+'.run = __test__%s__%s__run,'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln('};')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # create suite define names
|
|
|
|
|
+ if suite.defines:
|
|
|
|
|
+ f.writeln('const char *const __test__%s__define_names[] = {'
|
|
|
|
|
+ % suite.name)
|
|
|
|
|
+ for k in sorted(suite.defines):
|
|
|
|
|
+ f.writeln(4*' '+'"%s",' % k)
|
|
|
|
|
+ f.writeln('};')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # create suite struct
|
|
|
|
|
+ f.writeln('const struct test_suite __test__%s__suite = {'
|
|
|
|
|
+ % suite.name)
|
|
|
|
|
+ f.writeln(4*' '+'.id = "%s",' % suite.id())
|
|
|
|
|
+ f.writeln(4*' '+'.name = "%s",' % suite.name)
|
|
|
|
|
+ f.writeln(4*' '+'.path = "%s",' % suite.path)
|
|
|
|
|
+ f.writeln(4*' '+'.types = %s,'
|
|
|
|
|
+ % ' | '.join(filter(None, [
|
|
|
|
|
+ 'TEST_NORMAL' if suite.normal else None,
|
|
|
|
|
+ 'TEST_REENTRANT' if suite.reentrant else None])))
|
|
|
|
|
+ if suite.defines:
|
|
|
|
|
+ f.writeln(4*' '+'.define_names = __test__%s__define_names,'
|
|
|
|
|
+ % suite.name)
|
|
|
|
|
+ f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
|
|
|
|
|
+ f.writeln(4*' '+'.cases = (const struct test_case *const []){')
|
|
|
|
|
+ for case in suite.cases:
|
|
|
|
|
+ f.writeln(8*' '+'&__test__%s__%s__case,'
|
|
|
|
|
+ % (suite.name, case.name))
|
|
|
|
|
+ f.writeln(4*' '+'},')
|
|
|
|
|
+ f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
|
|
|
|
|
+ f.writeln('};')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ else:
|
|
|
|
|
+ # copy source
|
|
|
|
|
+ f.writeln('#line 1 "%s"' % args['source'])
|
|
|
|
|
+ with open(args['source']) as sf:
|
|
|
|
|
+ shutil.copyfileobj(sf, f)
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ f.write(SUITE_PROLOGUE)
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # write any internal tests
|
|
|
|
|
+ for suite in suites:
|
|
|
|
|
+ for case in suite.cases:
|
|
|
|
|
+ if (case.in_ is not None
|
|
|
|
|
+ and os.path.normpath(case.in_)
|
|
|
|
|
+ == os.path.normpath(args['source'])):
|
|
|
|
|
+ # write defines, but note we need to undef any
|
|
|
|
|
+ # new defines since we're in someone else's file
|
|
|
|
|
+ if suite.defines:
|
|
|
|
|
+ for i, define in enumerate(
|
|
|
|
|
+ sorted(suite.defines)):
|
|
|
|
|
+ f.writeln('#ifndef %s' % define)
|
|
|
|
|
+ f.writeln('#define %-24s test_define(%d)'
|
|
|
|
|
+ % (define, i))
|
|
|
|
|
+ f.writeln('#define __TEST__%s__NEEDS_UNDEF'
|
|
|
|
|
+ % define)
|
|
|
|
|
+ f.writeln('#endif')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ write_case_functions(f, suite, case)
|
|
|
|
|
+
|
|
|
|
|
+ if suite.defines:
|
|
|
|
|
+ for define in sorted(suite.defines):
|
|
|
|
|
+ f.writeln('#ifdef __TEST__%s__NEEDS_UNDEF'
|
|
|
|
|
+ % define)
|
|
|
|
|
+ f.writeln('#undef __TEST__%s__NEEDS_UNDEF'
|
|
|
|
|
+ % define)
|
|
|
|
|
+ f.writeln('#undef %s' % define)
|
|
|
|
|
+ f.writeln('#endif')
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+
|
|
|
|
|
+ # add suite info to test_runner.c
|
|
|
|
|
+ if args['source'] == 'runners/test_runner.c':
|
|
|
|
|
+ f.writeln()
|
|
|
|
|
+ for suite in suites:
|
|
|
|
|
+ f.writeln('extern const struct test_suite '
|
|
|
|
|
+ '__test__%s__suite;' % suite.name)
|
|
|
|
|
+ f.writeln('const struct test_suite *test_suites[] = {')
|
|
|
|
|
+ for suite in suites:
|
|
|
|
|
+ f.writeln(4*' '+'&__test__%s__suite,' % suite.name)
|
|
|
|
|
+ f.writeln('};')
|
|
|
|
|
+ f.writeln('const size_t test_suite_count = %d;'
|
|
|
|
|
+ % len(suites))
|
|
|
|
|
+
|
|
|
|
|
+def runner(**args):
|
|
|
|
|
+ cmd = args['runner'].copy()
|
|
|
|
|
+ cmd.extend(args.get('test_ids'))
|
|
|
|
|
+
|
|
|
|
|
+ # run under some external command?
|
|
|
|
|
+ cmd[:0] = args.get('exec', [])
|
|
|
|
|
+
|
|
|
|
|
+ # run under valgrind?
|
|
|
if args.get('valgrind'):
|
|
if args.get('valgrind'):
|
|
|
- classes.append(ValgrindTestCase)
|
|
|
|
|
- if not classes:
|
|
|
|
|
- classes = [TestCase]
|
|
|
|
|
-
|
|
|
|
|
- suites = []
|
|
|
|
|
- for testpath in args['test_paths']:
|
|
|
|
|
- # optionally specified test case/perm
|
|
|
|
|
- testpath, *filter = testpath.split('#')
|
|
|
|
|
- filter = [int(f) for f in filter]
|
|
|
|
|
-
|
|
|
|
|
- # figure out the suite's toml file
|
|
|
|
|
- if os.path.isdir(testpath):
|
|
|
|
|
- testpath = testpath + '/*.toml'
|
|
|
|
|
- elif os.path.isfile(testpath):
|
|
|
|
|
- testpath = testpath
|
|
|
|
|
- elif testpath.endswith('.toml'):
|
|
|
|
|
- testpath = TEST_PATHS + '/' + testpath
|
|
|
|
|
- else:
|
|
|
|
|
- testpath = TEST_PATHS + '/' + testpath + '.toml'
|
|
|
|
|
-
|
|
|
|
|
- # find tests
|
|
|
|
|
- for path in glob.glob(testpath):
|
|
|
|
|
- suites.append(TestSuite(path, classes, defines, filter, **args))
|
|
|
|
|
-
|
|
|
|
|
- # sort for reproducibility
|
|
|
|
|
- suites = sorted(suites)
|
|
|
|
|
-
|
|
|
|
|
- # generate permutations
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- suite.permute(**args)
|
|
|
|
|
-
|
|
|
|
|
- # build tests in parallel
|
|
|
|
|
- print('====== building ======')
|
|
|
|
|
- makefiles = []
|
|
|
|
|
- targets = []
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- makefile, target = suite.build(**args)
|
|
|
|
|
- makefiles.append(makefile)
|
|
|
|
|
- targets.append(target)
|
|
|
|
|
-
|
|
|
|
|
- cmd = (['make', '-f', 'Makefile'] +
|
|
|
|
|
- list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
|
|
|
|
|
- [target for target in targets])
|
|
|
|
|
- mpty, spty = pty.openpty()
|
|
|
|
|
|
|
+ cmd[:0] = filter(None, [
|
|
|
|
|
+ 'valgrind',
|
|
|
|
|
+ '--leak-check=full',
|
|
|
|
|
+ '--track-origins=yes',
|
|
|
|
|
+ '--error-exitcode=4',
|
|
|
|
|
+ '-q'])
|
|
|
|
|
+
|
|
|
|
|
+ # filter tests?
|
|
|
|
|
+ if args.get('normal'): cmd.append('-n')
|
|
|
|
|
+ if args.get('reentrant'): cmd.append('-r')
|
|
|
|
|
+ if args.get('geometry'):
|
|
|
|
|
+ cmd.append('-G%s' % args.get('geometry'))
|
|
|
|
|
+
|
|
|
|
|
+ # defines?
|
|
|
|
|
+ if args.get('define'):
|
|
|
|
|
+ for define in args.get('define'):
|
|
|
|
|
+ cmd.append('-D%s' % define)
|
|
|
|
|
+
|
|
|
|
|
+ return cmd
|
|
|
|
|
+
|
|
|
|
|
+def list_(**args):
|
|
|
|
|
+ cmd = runner(**args)
|
|
|
|
|
+ if args.get('summary'): cmd.append('--summary')
|
|
|
|
|
+ if args.get('list_suites'): cmd.append('--list-suites')
|
|
|
|
|
+ if args.get('list_cases'): cmd.append('--list-cases')
|
|
|
|
|
+ if args.get('list_paths'): cmd.append('--list-paths')
|
|
|
|
|
+ if args.get('list_defines'): cmd.append('--list-defines')
|
|
|
|
|
+ if args.get('list_geometries'): cmd.append('--list-geometries')
|
|
|
|
|
+
|
|
|
if args.get('verbose'):
|
|
if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
- proc = sp.Popen(cmd, stdout=spty, stderr=spty)
|
|
|
|
|
- os.close(spty)
|
|
|
|
|
- mpty = os.fdopen(mpty, 'r', 1)
|
|
|
|
|
- stdout = []
|
|
|
|
|
- while True:
|
|
|
|
|
- try:
|
|
|
|
|
- line = mpty.readline()
|
|
|
|
|
- except OSError as e:
|
|
|
|
|
- if e.errno == errno.EIO:
|
|
|
|
|
- break
|
|
|
|
|
- raise
|
|
|
|
|
- if not line:
|
|
|
|
|
- break;
|
|
|
|
|
- stdout.append(line)
|
|
|
|
|
- if args.get('verbose'):
|
|
|
|
|
- sys.stdout.write(line)
|
|
|
|
|
- # intercept warnings
|
|
|
|
|
- m = re.match(
|
|
|
|
|
- '^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'
|
|
|
|
|
- .format('(?:\033\[[\d;]*.| )*', 'warning'),
|
|
|
|
|
- line)
|
|
|
|
|
- if m and not args.get('verbose'):
|
|
|
|
|
- try:
|
|
|
|
|
- with open(m.group(1)) as f:
|
|
|
|
|
- lineno = int(m.group(2))
|
|
|
|
|
- line = next(it.islice(f, lineno-1, None)).strip('\n')
|
|
|
|
|
- sys.stdout.write(
|
|
|
|
|
- "\033[01m{path}:{lineno}:\033[01;35mwarning:\033[m "
|
|
|
|
|
- "{message}\n{line}\n\n".format(
|
|
|
|
|
- path=m.group(1), line=line, lineno=lineno,
|
|
|
|
|
- message=m.group(3)))
|
|
|
|
|
- except:
|
|
|
|
|
- pass
|
|
|
|
|
|
|
+ sys.exit(sp.call(cmd))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def find_cases(runner_, **args):
|
|
|
|
|
+ # query from runner
|
|
|
|
|
+ cmd = runner_ + ['--list-cases']
|
|
|
|
|
+ if args.get('verbose'):
|
|
|
|
|
+ print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
|
|
+ proc = sp.Popen(cmd,
|
|
|
|
|
+ stdout=sp.PIPE,
|
|
|
|
|
+ stderr=sp.PIPE if not args.get('verbose') else None,
|
|
|
|
|
+ universal_newlines=True,
|
|
|
|
|
+ errors='replace')
|
|
|
|
|
+ expected_suite_perms = co.defaultdict(lambda: 0)
|
|
|
|
|
+ expected_case_perms = co.defaultdict(lambda: 0)
|
|
|
|
|
+ expected_perms = 0
|
|
|
|
|
+ total_perms = 0
|
|
|
|
|
+ pattern = re.compile(
|
|
|
|
|
+ '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
|
|
|
|
|
+ '[^\s]+\s+(?P<filtered>\d+)/(?P<perms>\d+)')
|
|
|
|
|
+ # skip the first line
|
|
|
|
|
+ for line in it.islice(proc.stdout, 1, None):
|
|
|
|
|
+ m = pattern.match(line)
|
|
|
|
|
+ if m:
|
|
|
|
|
+ filtered = int(m.group('filtered'))
|
|
|
|
|
+ expected_suite_perms[m.group('suite')] += filtered
|
|
|
|
|
+ expected_case_perms[m.group('id')] += filtered
|
|
|
|
|
+ expected_perms += filtered
|
|
|
|
|
+ total_perms += int(m.group('perms'))
|
|
|
proc.wait()
|
|
proc.wait()
|
|
|
if proc.returncode != 0:
|
|
if proc.returncode != 0:
|
|
|
if not args.get('verbose'):
|
|
if not args.get('verbose'):
|
|
|
- for line in stdout:
|
|
|
|
|
|
|
+ for line in proc.stderr:
|
|
|
sys.stdout.write(line)
|
|
sys.stdout.write(line)
|
|
|
sys.exit(-1)
|
|
sys.exit(-1)
|
|
|
|
|
|
|
|
- print('built %d test suites, %d test cases, %d permutations' % (
|
|
|
|
|
- len(suites),
|
|
|
|
|
- sum(len(suite.cases) for suite in suites),
|
|
|
|
|
- sum(len(suite.perms) for suite in suites)))
|
|
|
|
|
|
|
+ return (
|
|
|
|
|
+ expected_suite_perms,
|
|
|
|
|
+ expected_case_perms,
|
|
|
|
|
+ expected_perms,
|
|
|
|
|
+ total_perms)
|
|
|
|
|
|
|
|
- total = 0
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- for perm in suite.perms:
|
|
|
|
|
- total += perm.shouldtest(**args)
|
|
|
|
|
- if total != sum(len(suite.perms) for suite in suites):
|
|
|
|
|
- print('filtered down to %d permutations' % total)
|
|
|
|
|
|
|
+def find_paths(runner_, **args):
|
|
|
|
|
+ # query from runner
|
|
|
|
|
+ cmd = runner_ + ['--list-paths']
|
|
|
|
|
+ if args.get('verbose'):
|
|
|
|
|
+ print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
|
|
+ proc = sp.Popen(cmd,
|
|
|
|
|
+ stdout=sp.PIPE,
|
|
|
|
|
+ stderr=sp.PIPE if not args.get('verbose') else None,
|
|
|
|
|
+ universal_newlines=True,
|
|
|
|
|
+ errors='replace')
|
|
|
|
|
+ paths = co.OrderedDict()
|
|
|
|
|
+ pattern = re.compile(
|
|
|
|
|
+ '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
|
|
|
|
|
+ '(?P<path>[^:]+):(?P<lineno>\d+)')
|
|
|
|
|
+ for line in proc.stdout:
|
|
|
|
|
+ m = pattern.match(line)
|
|
|
|
|
+ if m:
|
|
|
|
|
+ paths[m.group('id')] = (m.group('path'), int(m.group('lineno')))
|
|
|
|
|
+ proc.wait()
|
|
|
|
|
+ if proc.returncode != 0:
|
|
|
|
|
+ if not args.get('verbose'):
|
|
|
|
|
+ for line in proc.stderr:
|
|
|
|
|
+ sys.stdout.write(line)
|
|
|
|
|
+ sys.exit(-1)
|
|
|
|
|
|
|
|
- # only requested to build?
|
|
|
|
|
- if args.get('build'):
|
|
|
|
|
- return 0
|
|
|
|
|
|
|
+ return paths
|
|
|
|
|
|
|
|
- print('====== testing ======')
|
|
|
|
|
- try:
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- suite.test(**args)
|
|
|
|
|
- except TestFailure:
|
|
|
|
|
- pass
|
|
|
|
|
|
|
+def find_defines(runner_, **args):
|
|
|
|
|
+ # query from runner
|
|
|
|
|
+ cmd = runner_ + ['--list-defines']
|
|
|
|
|
+ if args.get('verbose'):
|
|
|
|
|
+ print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
|
|
+ proc = sp.Popen(cmd,
|
|
|
|
|
+ stdout=sp.PIPE,
|
|
|
|
|
+ stderr=sp.PIPE if not args.get('verbose') else None,
|
|
|
|
|
+ universal_newlines=True,
|
|
|
|
|
+ errors='replace')
|
|
|
|
|
+ defines = co.OrderedDict()
|
|
|
|
|
+ pattern = re.compile(
|
|
|
|
|
+ '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
|
|
|
|
|
+ '(?P<defines>(?:\w+=\w+\s*)+)')
|
|
|
|
|
+ for line in proc.stdout:
|
|
|
|
|
+ m = pattern.match(line)
|
|
|
|
|
+ if m:
|
|
|
|
|
+ defines[m.group('id')] = {k: v
|
|
|
|
|
+ for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))}
|
|
|
|
|
+ proc.wait()
|
|
|
|
|
+ if proc.returncode != 0:
|
|
|
|
|
+ if not args.get('verbose'):
|
|
|
|
|
+ for line in proc.stderr:
|
|
|
|
|
+ sys.stdout.write(line)
|
|
|
|
|
+ sys.exit(-1)
|
|
|
|
|
|
|
|
- print('====== results ======')
|
|
|
|
|
- passed = 0
|
|
|
|
|
- failed = 0
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- for perm in suite.perms:
|
|
|
|
|
- if perm.result == PASS:
|
|
|
|
|
- passed += 1
|
|
|
|
|
- elif isinstance(perm.result, TestFailure):
|
|
|
|
|
- sys.stdout.write(
|
|
|
|
|
- "\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m "
|
|
|
|
|
- "{perm} failed\n".format(
|
|
|
|
|
- perm=perm, path=perm.suite.path, lineno=perm.lineno,
|
|
|
|
|
- returncode=perm.result.returncode or 0))
|
|
|
|
|
- if perm.result.stdout:
|
|
|
|
|
- if perm.result.assert_:
|
|
|
|
|
- stdout = perm.result.stdout[:-1]
|
|
|
|
|
- else:
|
|
|
|
|
- stdout = perm.result.stdout
|
|
|
|
|
- for line in stdout[-5:]:
|
|
|
|
|
- sys.stdout.write(line)
|
|
|
|
|
- if perm.result.assert_:
|
|
|
|
|
- sys.stdout.write(
|
|
|
|
|
- "\033[01m{path}:{lineno}:\033[01;31massert:\033[m "
|
|
|
|
|
- "{message}\n{line}\n".format(
|
|
|
|
|
- **perm.result.assert_))
|
|
|
|
|
- sys.stdout.write('\n')
|
|
|
|
|
- failed += 1
|
|
|
|
|
-
|
|
|
|
|
- if args.get('coverage'):
|
|
|
|
|
- # collect coverage info
|
|
|
|
|
- # why -j1? lcov doesn't work in parallel because of gcov limitations
|
|
|
|
|
- cmd = (['make', '-j1', '-f', 'Makefile'] +
|
|
|
|
|
- list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
|
|
|
|
|
- (['COVERAGETARGET=%s' % args['coverage']]
|
|
|
|
|
- if isinstance(args['coverage'], str) else []) +
|
|
|
|
|
- [suite.path + '.info' for suite in suites
|
|
|
|
|
- if any(perm.result == PASS for perm in suite.perms)])
|
|
|
|
|
|
|
+ return defines
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+class TestFailure(Exception):
|
|
|
|
|
+ def __init__(self, id, returncode, output, assert_=None):
|
|
|
|
|
+ self.id = id
|
|
|
|
|
+ self.returncode = returncode
|
|
|
|
|
+ self.output = output
|
|
|
|
|
+ self.assert_ = assert_
|
|
|
|
|
+
|
|
|
|
|
+def run_stage(name, runner_, **args):
|
|
|
|
|
+ # get expected suite/case/perm counts
|
|
|
|
|
+ expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
|
|
|
|
|
+ find_cases(runner_, **args))
|
|
|
|
|
+
|
|
|
|
|
+ passed_suite_perms = co.defaultdict(lambda: 0)
|
|
|
|
|
+ passed_case_perms = co.defaultdict(lambda: 0)
|
|
|
|
|
+ passed_perms = 0
|
|
|
|
|
+ failures = []
|
|
|
|
|
+ killed = False
|
|
|
|
|
+
|
|
|
|
|
+ pattern = re.compile('^(?:'
|
|
|
|
|
+ '(?P<op>running|finished|skipped) '
|
|
|
|
|
+ '(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
|
|
|
|
|
+ '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
|
|
|
|
|
+ ' *(?P<message>.*)' ')$')
|
|
|
|
|
+ locals = th.local()
|
|
|
|
|
+ children = set()
|
|
|
|
|
+
|
|
|
|
|
+ def run_runner(runner_):
|
|
|
|
|
+ nonlocal passed_suite_perms
|
|
|
|
|
+ nonlocal passed_case_perms
|
|
|
|
|
+ nonlocal passed_perms
|
|
|
|
|
+ nonlocal locals
|
|
|
|
|
+
|
|
|
|
|
+ # run the tests!
|
|
|
|
|
+ cmd = runner_.copy()
|
|
|
|
|
+ if args.get('disk'):
|
|
|
|
|
+ cmd.append('--disk=%s' % args['disk'])
|
|
|
|
|
+ if args.get('trace'):
|
|
|
|
|
+ cmd.append('--trace=%s' % args['trace'])
|
|
|
if args.get('verbose'):
|
|
if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
- proc = sp.Popen(cmd,
|
|
|
|
|
- stdout=sp.PIPE if not args.get('verbose') else None,
|
|
|
|
|
- stderr=sp.STDOUT if not args.get('verbose') else None,
|
|
|
|
|
- universal_newlines=True)
|
|
|
|
|
- stdout = []
|
|
|
|
|
- for line in proc.stdout:
|
|
|
|
|
- stdout.append(line)
|
|
|
|
|
|
|
+ mpty, spty = pty.openpty()
|
|
|
|
|
+ proc = sp.Popen(cmd, stdout=spty, stderr=spty)
|
|
|
|
|
+ os.close(spty)
|
|
|
|
|
+ children.add(proc)
|
|
|
|
|
+ mpty = os.fdopen(mpty, 'r', 1)
|
|
|
|
|
+ if args.get('output'):
|
|
|
|
|
+ output = openio(args['output'], 'w')
|
|
|
|
|
+
|
|
|
|
|
+ last_id = None
|
|
|
|
|
+ last_output = []
|
|
|
|
|
+ last_assert = None
|
|
|
|
|
+ try:
|
|
|
|
|
+ while True:
|
|
|
|
|
+ # parse a line for state changes
|
|
|
|
|
+ try:
|
|
|
|
|
+ line = mpty.readline()
|
|
|
|
|
+ except OSError as e:
|
|
|
|
|
+ if e.errno == errno.EIO:
|
|
|
|
|
+ break
|
|
|
|
|
+ raise
|
|
|
|
|
+ if not line:
|
|
|
|
|
+ break
|
|
|
|
|
+ last_output.append(line)
|
|
|
|
|
+ if args.get('output'):
|
|
|
|
|
+ output.write(line)
|
|
|
|
|
+ elif args.get('verbose'):
|
|
|
|
|
+ sys.stdout.write(line)
|
|
|
|
|
+
|
|
|
|
|
+ m = pattern.match(line)
|
|
|
|
|
+ if m:
|
|
|
|
|
+ op = m.group('op') or m.group('op_')
|
|
|
|
|
+ if op == 'running':
|
|
|
|
|
+ locals.seen_perms += 1
|
|
|
|
|
+ last_id = m.group('id')
|
|
|
|
|
+ last_output = []
|
|
|
|
|
+ last_assert = None
|
|
|
|
|
+ elif op == 'finished':
|
|
|
|
|
+ passed_suite_perms[m.group('suite')] += 1
|
|
|
|
|
+ passed_case_perms[m.group('case')] += 1
|
|
|
|
|
+ passed_perms += 1
|
|
|
|
|
+ elif op == 'skipped':
|
|
|
|
|
+ locals.seen_perms += 1
|
|
|
|
|
+ elif op == 'assert':
|
|
|
|
|
+ last_assert = (
|
|
|
|
|
+ m.group('path'),
|
|
|
|
|
+ int(m.group('lineno')),
|
|
|
|
|
+ m.group('message'))
|
|
|
|
|
+ # go ahead and kill the process, aborting takes a while
|
|
|
|
|
+ if args.get('keep_going'):
|
|
|
|
|
+ proc.kill()
|
|
|
|
|
+ except KeyboardInterrupt:
|
|
|
|
|
+ raise TestFailure(last_id, 1, last_output)
|
|
|
|
|
+ finally:
|
|
|
|
|
+ children.remove(proc)
|
|
|
|
|
+ mpty.close()
|
|
|
|
|
+ if args.get('output'):
|
|
|
|
|
+ output.close()
|
|
|
|
|
+
|
|
|
proc.wait()
|
|
proc.wait()
|
|
|
if proc.returncode != 0:
|
|
if proc.returncode != 0:
|
|
|
|
|
+ raise TestFailure(
|
|
|
|
|
+ last_id,
|
|
|
|
|
+ proc.returncode,
|
|
|
|
|
+ last_output,
|
|
|
|
|
+ last_assert)
|
|
|
|
|
+
|
|
|
|
|
+ def run_job(runner, start=None, step=None):
|
|
|
|
|
+ nonlocal failures
|
|
|
|
|
+ nonlocal locals
|
|
|
|
|
+
|
|
|
|
|
+ start = start or 0
|
|
|
|
|
+ step = step or 1
|
|
|
|
|
+ while start < total_perms:
|
|
|
|
|
+ runner_ = runner.copy()
|
|
|
|
|
+ if start is not None:
|
|
|
|
|
+ runner_.append('--start=%d' % start)
|
|
|
|
|
+ if step is not None:
|
|
|
|
|
+ runner_.append('--step=%d' % step)
|
|
|
|
|
+ if args.get('isolate') or args.get('valgrind'):
|
|
|
|
|
+ runner_.append('--stop=%d' % (start+step))
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ # run the tests
|
|
|
|
|
+ locals.seen_perms = 0
|
|
|
|
|
+ run_runner(runner_)
|
|
|
|
|
+ assert locals.seen_perms > 0
|
|
|
|
|
+ start += locals.seen_perms*step
|
|
|
|
|
+
|
|
|
|
|
+ except TestFailure as failure:
|
|
|
|
|
+ # race condition for multiple failures?
|
|
|
|
|
+ if failures and not args.get('keep_going'):
|
|
|
|
|
+ break
|
|
|
|
|
+
|
|
|
|
|
+ failures.append(failure)
|
|
|
|
|
+
|
|
|
|
|
+ if args.get('keep_going') and not killed:
|
|
|
|
|
+ # resume after failed test
|
|
|
|
|
+ assert locals.seen_perms > 0
|
|
|
|
|
+ start += locals.seen_perms*step
|
|
|
|
|
+ continue
|
|
|
|
|
+ else:
|
|
|
|
|
+ # stop other tests
|
|
|
|
|
+ for child in children.copy():
|
|
|
|
|
+ child.kill()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ # parallel jobs?
|
|
|
|
|
+ runners = []
|
|
|
|
|
+ if 'jobs' in args:
|
|
|
|
|
+ for job in range(args['jobs']):
|
|
|
|
|
+ runners.append(th.Thread(
|
|
|
|
|
+ target=run_job, args=(runner_, job, args['jobs'])))
|
|
|
|
|
+ else:
|
|
|
|
|
+ runners.append(th.Thread(
|
|
|
|
|
+ target=run_job, args=(runner_, None, None)))
|
|
|
|
|
+
|
|
|
|
|
+ for r in runners:
|
|
|
|
|
+ r.start()
|
|
|
|
|
+
|
|
|
|
|
+ needs_newline = False
|
|
|
|
|
+ try:
|
|
|
|
|
+ while any(r.is_alive() for r in runners):
|
|
|
|
|
+ time.sleep(0.01)
|
|
|
|
|
+
|
|
|
if not args.get('verbose'):
|
|
if not args.get('verbose'):
|
|
|
- for line in stdout:
|
|
|
|
|
- sys.stdout.write(line)
|
|
|
|
|
- sys.exit(-1)
|
|
|
|
|
|
|
+ sys.stdout.write('\r\x1b[K'
|
|
|
|
|
+ 'running \x1b[%dm%s:\x1b[m %s '
|
|
|
|
|
+ % (32 if not failures else 31,
|
|
|
|
|
+ name,
|
|
|
|
|
+ ', '.join(filter(None, [
|
|
|
|
|
+ '%d/%d suites' % (
|
|
|
|
|
+ sum(passed_suite_perms[k] == v
|
|
|
|
|
+ for k, v in expected_suite_perms.items()),
|
|
|
|
|
+ len(expected_suite_perms))
|
|
|
|
|
+ if (not args.get('by_suites')
|
|
|
|
|
+ and not args.get('by_cases')) else None,
|
|
|
|
|
+ '%d/%d cases' % (
|
|
|
|
|
+ sum(passed_case_perms[k] == v
|
|
|
|
|
+ for k, v in expected_case_perms.items()),
|
|
|
|
|
+ len(expected_case_perms))
|
|
|
|
|
+ if not args.get('by_cases') else None,
|
|
|
|
|
+ '%d/%d perms' % (passed_perms, expected_perms),
|
|
|
|
|
+ '\x1b[31m%d/%d failures\x1b[m'
|
|
|
|
|
+ % (len(failures), expected_perms)
|
|
|
|
|
+ if failures else None]))))
|
|
|
|
|
+ sys.stdout.flush()
|
|
|
|
|
+ needs_newline = True
|
|
|
|
|
+ except KeyboardInterrupt:
|
|
|
|
|
+ # this is handled by the runner threads, we just
|
|
|
|
|
+ # need to not abort here
|
|
|
|
|
+ killed = True
|
|
|
|
|
+ finally:
|
|
|
|
|
+ if needs_newline:
|
|
|
|
|
+ print()
|
|
|
|
|
+
|
|
|
|
|
+ for r in runners:
|
|
|
|
|
+ r.join()
|
|
|
|
|
+
|
|
|
|
|
+ return (
|
|
|
|
|
+ expected_perms,
|
|
|
|
|
+ passed_perms,
|
|
|
|
|
+ failures,
|
|
|
|
|
+ killed)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def run(**args):
|
|
|
|
|
+ start = time.time()
|
|
|
|
|
+
|
|
|
|
|
+ runner_ = runner(**args)
|
|
|
|
|
+ print('using runner: %s'
|
|
|
|
|
+ % ' '.join(shlex.quote(c) for c in runner_))
|
|
|
|
|
+ expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
|
|
|
|
|
+ find_cases(runner_, **args))
|
|
|
|
|
+ print('found %d suites, %d cases, %d/%d permutations'
|
|
|
|
|
+ % (len(expected_suite_perms),
|
|
|
|
|
+ len(expected_case_perms),
|
|
|
|
|
+ expected_perms,
|
|
|
|
|
+ total_perms))
|
|
|
|
|
+ print()
|
|
|
|
|
+
|
|
|
|
|
+ expected = 0
|
|
|
|
|
+ passed = 0
|
|
|
|
|
+ failures = []
|
|
|
|
|
+ for type, by in it.product(
|
|
|
|
|
+ ['normal', 'reentrant'],
|
|
|
|
|
+ expected_case_perms.keys() if args.get('by_cases')
|
|
|
|
|
+ else expected_suite_perms.keys() if args.get('by_suites')
|
|
|
|
|
+ else [None]):
|
|
|
|
|
+ # rebuild runner for each stage to override test identifier if needed
|
|
|
|
|
+ stage_runner = runner(**args | {
|
|
|
|
|
+ 'test_ids': [by] if by is not None else args.get('test_ids', []),
|
|
|
|
|
+ 'normal': type == 'normal',
|
|
|
|
|
+ 'reentrant': type == 'reentrant'})
|
|
|
|
|
+
|
|
|
|
|
+ # spawn jobs for stage
|
|
|
|
|
+ expected_, passed_, failures_, killed = run_stage(
|
|
|
|
|
+ '%s %s' % (type, by or 'tests'), stage_runner, **args)
|
|
|
|
|
+ expected += expected_
|
|
|
|
|
+ passed += passed_
|
|
|
|
|
+ failures.extend(failures_)
|
|
|
|
|
+ if (failures and not args.get('keep_going')) or killed:
|
|
|
|
|
+ break
|
|
|
|
|
+
|
|
|
|
|
+ # show summary
|
|
|
|
|
+ print()
|
|
|
|
|
+ print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs'
|
|
|
|
|
+ % (32 if not failures else 31,
|
|
|
|
|
+ passed, expected, len(failures), expected,
|
|
|
|
|
+ time.time()-start))
|
|
|
|
|
+ print()
|
|
|
|
|
+
|
|
|
|
|
+ # print each failure
|
|
|
|
|
+ if failures:
|
|
|
|
|
+ # get some extra info from runner
|
|
|
|
|
+ runner_paths = find_paths(runner_, **args)
|
|
|
|
|
+ runner_defines = find_defines(runner_, **args)
|
|
|
|
|
+
|
|
|
|
|
+ for failure in failures:
|
|
|
|
|
+ # show summary of failure
|
|
|
|
|
+ path, lineno = runner_paths[testcase(failure.id)]
|
|
|
|
|
+ defines = runner_defines[failure.id]
|
|
|
|
|
+
|
|
|
|
|
+ print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed'
|
|
|
|
|
+ % (path, lineno, failure.id,
|
|
|
|
|
+ ' (%s)' % ', '.join(
|
|
|
|
|
+ '%s=%s' % (k, v) for k, v in defines.items())
|
|
|
|
|
+ if defines else ''))
|
|
|
|
|
+
|
|
|
|
|
+ if failure.output:
|
|
|
|
|
+ output = failure.output
|
|
|
|
|
+ if failure.assert_ is not None:
|
|
|
|
|
+ output = output[:-1]
|
|
|
|
|
+ for line in output[-5:]:
|
|
|
|
|
+ sys.stdout.write(line)
|
|
|
|
|
+
|
|
|
|
|
+ if failure.assert_ is not None:
|
|
|
|
|
+ path, lineno, message = failure.assert_
|
|
|
|
|
+ print('\x1b[01m%s:%d:\x1b[01;31massert:\x1b[m %s'
|
|
|
|
|
+ % (path, lineno, message))
|
|
|
|
|
+ with open(path) as f:
|
|
|
|
|
+ line = next(it.islice(f, lineno-1, None)).strip('\n')
|
|
|
|
|
+ print(line)
|
|
|
|
|
+ print()
|
|
|
|
|
+
|
|
|
|
|
+ # drop into gdb?
|
|
|
|
|
+ if failures and (args.get('gdb')
|
|
|
|
|
+ or args.get('gdb_case')
|
|
|
|
|
+ or args.get('gdb_main')):
|
|
|
|
|
+ failure = failures[0]
|
|
|
|
|
+ runner_ = runner(**args | {'test_ids': [failure.id]})
|
|
|
|
|
+
|
|
|
|
|
+ if args.get('gdb_main'):
|
|
|
|
|
+ cmd = ['gdb',
|
|
|
|
|
+ '-ex', 'break main',
|
|
|
|
|
+ '-ex', 'run',
|
|
|
|
|
+ '--args'] + runner_
|
|
|
|
|
+ elif args.get('gdb_case'):
|
|
|
|
|
+ path, lineno = runner_paths[testcase(failure.id)]
|
|
|
|
|
+ cmd = ['gdb',
|
|
|
|
|
+ '-ex', 'break %s:%d' % (path, lineno),
|
|
|
|
|
+ '-ex', 'run',
|
|
|
|
|
+ '--args'] + runner_
|
|
|
|
|
+ elif failure.assert_ is not None:
|
|
|
|
|
+ cmd = ['gdb',
|
|
|
|
|
+ '-ex', 'run',
|
|
|
|
|
+ '-ex', 'frame function raise',
|
|
|
|
|
+ '-ex', 'up 2',
|
|
|
|
|
+ '--args'] + runner_
|
|
|
|
|
+ else:
|
|
|
|
|
+ cmd = ['gdb',
|
|
|
|
|
+ '-ex', 'run',
|
|
|
|
|
+ '--args'] + runner_
|
|
|
|
|
+
|
|
|
|
|
+ # exec gdb interactively
|
|
|
|
|
+ if args.get('verbose'):
|
|
|
|
|
+ print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
|
|
+ os.execvp(cmd[0], cmd)
|
|
|
|
|
+
|
|
|
|
|
+ return 1 if failures else 0
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def main(**args):
|
|
|
|
|
+ if args.get('compile'):
|
|
|
|
|
+ compile(**args)
|
|
|
|
|
+ elif (args.get('summary')
|
|
|
|
|
+ or args.get('list_suites')
|
|
|
|
|
+ or args.get('list_cases')
|
|
|
|
|
+ or args.get('list_paths')
|
|
|
|
|
+ or args.get('list_defines')
|
|
|
|
|
+ or args.get('list_geometries')
|
|
|
|
|
+ or args.get('list_defaults')):
|
|
|
|
|
+ list_(**args)
|
|
|
|
|
+ else:
|
|
|
|
|
+ run(**args)
|
|
|
|
|
|
|
|
- if args.get('gdb'):
|
|
|
|
|
- failure = None
|
|
|
|
|
- for suite in suites:
|
|
|
|
|
- for perm in suite.perms:
|
|
|
|
|
- if isinstance(perm.result, TestFailure):
|
|
|
|
|
- failure = perm.result
|
|
|
|
|
- if failure is not None:
|
|
|
|
|
- print('======= gdb ======')
|
|
|
|
|
- # drop into gdb
|
|
|
|
|
- failure.case.test(failure=failure, **args)
|
|
|
|
|
- sys.exit(0)
|
|
|
|
|
-
|
|
|
|
|
- print('tests passed %d/%d (%.1f%%)' % (passed, total,
|
|
|
|
|
- 100*(passed/total if total else 1.0)))
|
|
|
|
|
- print('tests failed %d/%d (%.1f%%)' % (failed, total,
|
|
|
|
|
- 100*(failed/total if total else 1.0)))
|
|
|
|
|
- return 1 if failed > 0 else 0
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if __name__ == "__main__":
|
|
|
import argparse
|
|
import argparse
|
|
|
|
|
+ import sys
|
|
|
parser = argparse.ArgumentParser(
|
|
parser = argparse.ArgumentParser(
|
|
|
- description="Run parameterized tests in various configurations.")
|
|
|
|
|
- parser.add_argument('test_paths', nargs='*', default=[TEST_PATHS],
|
|
|
|
|
- help="Description of test(s) to run. By default, this is all tests \
|
|
|
|
|
- found in the \"{0}\" directory. Here, you can specify a different \
|
|
|
|
|
- directory of tests, a specific file, a suite by name, and even \
|
|
|
|
|
- specific test cases and permutations. For example \
|
|
|
|
|
- \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TEST_PATHS))
|
|
|
|
|
- parser.add_argument('-D', action='append', default=[],
|
|
|
|
|
- help="Overriding parameter definitions.")
|
|
|
|
|
|
|
+ description="Build and run tests.",
|
|
|
|
|
+ conflict_handler='resolve')
|
|
|
|
|
+ parser.add_argument('test_ids', nargs='*',
|
|
|
|
|
+ help="Description of testis to run. May be a directory, path, or \
|
|
|
|
|
+ test identifier. Test identifiers are of the form \
|
|
|
|
|
+ <suite_name>#<case_name>#<permutation>, but suffixes can be \
|
|
|
|
|
+ dropped to run any matching tests. Defaults to %r." % TEST_PATHS)
|
|
|
parser.add_argument('-v', '--verbose', action='store_true',
|
|
parser.add_argument('-v', '--verbose', action='store_true',
|
|
|
- help="Output everything that is happening.")
|
|
|
|
|
- parser.add_argument('-k', '--keep-going', action='store_true',
|
|
|
|
|
- help="Run all tests instead of stopping on first error. Useful for CI.")
|
|
|
|
|
- parser.add_argument('-p', '--persist', choices=['erase', 'noerase'],
|
|
|
|
|
- nargs='?', const='erase',
|
|
|
|
|
- help="Store disk image in a file.")
|
|
|
|
|
- parser.add_argument('-b', '--build', action='store_true',
|
|
|
|
|
- help="Only build the tests, do not execute.")
|
|
|
|
|
- parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'],
|
|
|
|
|
- nargs='?', const='assert',
|
|
|
|
|
|
|
+ help="Output commands that run behind the scenes.")
|
|
|
|
|
+ # test flags
|
|
|
|
|
+ test_parser = parser.add_argument_group('test options')
|
|
|
|
|
+ test_parser.add_argument('-Y', '--summary', action='store_true',
|
|
|
|
|
+ help="Show quick summary.")
|
|
|
|
|
+ test_parser.add_argument('-l', '--list-suites', action='store_true',
|
|
|
|
|
+ help="List test suites.")
|
|
|
|
|
+ test_parser.add_argument('-L', '--list-cases', action='store_true',
|
|
|
|
|
+ help="List test cases.")
|
|
|
|
|
+ test_parser.add_argument('--list-paths', action='store_true',
|
|
|
|
|
+ help="List the path for each test case.")
|
|
|
|
|
+ test_parser.add_argument('--list-defines', action='store_true',
|
|
|
|
|
+ help="List the defines for each test permutation.")
|
|
|
|
|
+ test_parser.add_argument('--list-geometries', action='store_true',
|
|
|
|
|
+ help="List the disk geometries used for testing.")
|
|
|
|
|
+ test_parser.add_argument('--list-defaults', action='store_true',
|
|
|
|
|
+ help="List the default defines in this test-runner.")
|
|
|
|
|
+ test_parser.add_argument('-D', '--define', action='append',
|
|
|
|
|
+ help="Override a test define.")
|
|
|
|
|
+ test_parser.add_argument('-G', '--geometry',
|
|
|
|
|
+ help="Filter by geometry.")
|
|
|
|
|
+ test_parser.add_argument('-n', '--normal', action='store_true',
|
|
|
|
|
+ help="Filter for normal tests. Can be combined.")
|
|
|
|
|
+ test_parser.add_argument('-r', '--reentrant', action='store_true',
|
|
|
|
|
+ help="Filter for reentrant tests. Can be combined.")
|
|
|
|
|
+ test_parser.add_argument('-d', '--disk',
|
|
|
|
|
+ help="Use this file as the disk.")
|
|
|
|
|
+ test_parser.add_argument('-t', '--trace',
|
|
|
|
|
+ help="Redirect trace output to this file.")
|
|
|
|
|
+ test_parser.add_argument('-o', '--output',
|
|
|
|
|
+ help="Redirect stdout and stderr to this file.")
|
|
|
|
|
+ test_parser.add_argument('--runner', default=[RUNNER_PATH],
|
|
|
|
|
+ type=lambda x: x.split(),
|
|
|
|
|
+ help="Path to runner, defaults to %r" % RUNNER_PATH)
|
|
|
|
|
+ test_parser.add_argument('-j', '--jobs', nargs='?', type=int,
|
|
|
|
|
+ const=len(os.sched_getaffinity(0)),
|
|
|
|
|
+ help="Number of parallel runners to run.")
|
|
|
|
|
+ test_parser.add_argument('-k', '--keep-going', action='store_true',
|
|
|
|
|
+ help="Don't stop on first error.")
|
|
|
|
|
+ test_parser.add_argument('-i', '--isolate', action='store_true',
|
|
|
|
|
+ help="Run each test permutation in a separate process.")
|
|
|
|
|
+ test_parser.add_argument('-b', '--by-suites', action='store_true',
|
|
|
|
|
+ help="Step through tests by suite.")
|
|
|
|
|
+ test_parser.add_argument('-B', '--by-cases', action='store_true',
|
|
|
|
|
+ help="Step through tests by case.")
|
|
|
|
|
+ test_parser.add_argument('--gdb', action='store_true',
|
|
|
help="Drop into gdb on test failure.")
|
|
help="Drop into gdb on test failure.")
|
|
|
- parser.add_argument('--no-internal', action='store_true',
|
|
|
|
|
- help="Don't run tests that require internal knowledge.")
|
|
|
|
|
- parser.add_argument('-n', '--normal', action='store_true',
|
|
|
|
|
- help="Run tests normally.")
|
|
|
|
|
- parser.add_argument('-r', '--reentrant', action='store_true',
|
|
|
|
|
- help="Run reentrant tests with simulated power-loss.")
|
|
|
|
|
- parser.add_argument('--valgrind', action='store_true',
|
|
|
|
|
- help="Run non-leaky tests under valgrind to check for memory leaks.")
|
|
|
|
|
- parser.add_argument('--exec', default=[], type=lambda e: e.split(),
|
|
|
|
|
- help="Run tests with another executable prefixed on the command line.")
|
|
|
|
|
- parser.add_argument('--disk',
|
|
|
|
|
- help="Specify a file to use for persistent/reentrant tests.")
|
|
|
|
|
- parser.add_argument('--coverage', type=lambda x: x if x else True,
|
|
|
|
|
- nargs='?', const='',
|
|
|
|
|
- help="Collect coverage information during testing. This uses lcov/gcov \
|
|
|
|
|
- to accumulate coverage information into *.info files. May also \
|
|
|
|
|
- a path to a *.info file to accumulate coverage info into.")
|
|
|
|
|
- parser.add_argument('--build-dir',
|
|
|
|
|
- help="Build relative to the specified directory instead of the \
|
|
|
|
|
- current directory.")
|
|
|
|
|
-
|
|
|
|
|
- sys.exit(main(**vars(parser.parse_args())))
|
|
|
|
|
|
|
+ test_parser.add_argument('--gdb-case', action='store_true',
|
|
|
|
|
+ help="Drop into gdb on test failure but stop at the beginning \
|
|
|
|
|
+ of the failing test case.")
|
|
|
|
|
+ test_parser.add_argument('--gdb-main', action='store_true',
|
|
|
|
|
+ help="Drop into gdb on test failure but stop at the beginning \
|
|
|
|
|
+ of main.")
|
|
|
|
|
+ test_parser.add_argument('--valgrind', action='store_true',
|
|
|
|
|
+ help="Run under Valgrind to find memory errors. Implicitly sets \
|
|
|
|
|
+ --isolate.")
|
|
|
|
|
+ test_parser.add_argument('--exec', default=[], type=lambda e: e.split(),
|
|
|
|
|
+ help="Run under another executable.")
|
|
|
|
|
+ # compilation flags
|
|
|
|
|
+ comp_parser = parser.add_argument_group('compilation options')
|
|
|
|
|
+ comp_parser.add_argument('-c', '--compile', action='store_true',
|
|
|
|
|
+ help="Compile a test suite or source file.")
|
|
|
|
|
+ comp_parser.add_argument('-s', '--source',
|
|
|
|
|
+ help="Source file to compile, possibly injecting internal tests.")
|
|
|
|
|
+ comp_parser.add_argument('-o', '--output',
|
|
|
|
|
+ help="Output file.")
|
|
|
|
|
+ # TODO apply this to other scripts?
|
|
|
|
|
+ sys.exit(main(**{k: v
|
|
|
|
|
+ for k, v in vars(parser.parse_args()).items()
|
|
|
|
|
+ if v is not None}))
|