|
|
@@ -34,14 +34,18 @@ $(foreach target,$(SRC),$(eval $(FLATTEN)))
|
|
|
%(path)s.test: %(path)s.test.o \\
|
|
|
$(foreach t,$(subst /,.,$(SRC:.c=.o)),%(path)s.$t)
|
|
|
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
|
|
|
+
|
|
|
+# needed in case builddir is different
|
|
|
+%(path)s%%.o: %(path)s%%.c
|
|
|
+ $(CC) -c -MMD $(CFLAGS) $< -o $@
|
|
|
"""
|
|
|
COVERAGE_RULES = """
|
|
|
%(path)s.test: override CFLAGS += -fprofile-arcs -ftest-coverage
|
|
|
|
|
|
# delete lingering coverage
|
|
|
-%(path)s.test: | %(path)s.clean
|
|
|
-.PHONY: %(path)s.clean
|
|
|
-%(path)s.clean:
|
|
|
+%(path)s.test: | %(path)s.info.clean
|
|
|
+.PHONY: %(path)s.info.clean
|
|
|
+%(path)s.info.clean:
|
|
|
rm -f %(path)s*.gcda
|
|
|
|
|
|
# accumulate coverage info
|
|
|
@@ -52,10 +56,11 @@ COVERAGE_RULES = """
|
|
|
--rc 'geninfo_adjust_src_path=$(shell pwd)' \\
|
|
|
-o $@)
|
|
|
$(LCOV) -e $@ $(addprefix /,$(SRC)) -o $@
|
|
|
-
|
|
|
-.PHONY: %(path)s.cumul.info
|
|
|
-%(path)s.cumul.info: %(path)s.info
|
|
|
- $(LCOV) -a $< $(addprefix -a ,$(wildcard $@)) -o $@
|
|
|
+ifdef COVERAGETARGET
|
|
|
+ $(strip $(LCOV) -a $@ \\
|
|
|
+ $(addprefix -a ,$(wildcard $(COVERAGETARGET))) \\
|
|
|
+ -o $(COVERAGETARGET))
|
|
|
+endif
|
|
|
"""
|
|
|
GLOBALS = """
|
|
|
//////////////// AUTOGENERATED TEST ////////////////
|
|
|
@@ -142,6 +147,8 @@ class TestCase:
|
|
|
self.if_ = config.get('if', None)
|
|
|
self.in_ = config.get('in', None)
|
|
|
|
|
|
+ self.result = None
|
|
|
+
|
|
|
def __str__(self):
|
|
|
if hasattr(self, 'permno'):
|
|
|
if any(k not in self.case.defines for k in self.defines):
|
|
|
@@ -202,7 +209,7 @@ class TestCase:
|
|
|
len(self.filter) >= 2 and
|
|
|
self.filter[1] != self.permno):
|
|
|
return False
|
|
|
- elif args.get('no_internal', False) and self.in_ is not None:
|
|
|
+ elif args.get('no_internal') and self.in_ is not None:
|
|
|
return False
|
|
|
elif self.if_ is not None:
|
|
|
if_ = self.if_
|
|
|
@@ -236,7 +243,7 @@ class TestCase:
|
|
|
try:
|
|
|
with open(disk, 'w') as f:
|
|
|
f.truncate(0)
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
print('truncate --size=0', disk)
|
|
|
except FileNotFoundError:
|
|
|
pass
|
|
|
@@ -260,14 +267,14 @@ class TestCase:
|
|
|
'-ex', 'r'])
|
|
|
ncmd.extend(['--args'] + cmd)
|
|
|
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in ncmd))
|
|
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
|
sys.exit(sp.call(ncmd))
|
|
|
|
|
|
# run test case!
|
|
|
mpty, spty = pty.openpty()
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
proc = sp.Popen(cmd, stdout=spty, stderr=spty)
|
|
|
os.close(spty)
|
|
|
@@ -283,7 +290,7 @@ class TestCase:
|
|
|
break
|
|
|
raise
|
|
|
stdout.append(line)
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
sys.stdout.write(line)
|
|
|
# intercept asserts
|
|
|
m = re.match(
|
|
|
@@ -322,7 +329,7 @@ class ValgrindTestCase(TestCase):
|
|
|
return not self.leaky and super().shouldtest(**args)
|
|
|
|
|
|
def test(self, exec=[], **args):
|
|
|
- verbose = args.get('verbose', False)
|
|
|
+ verbose = args.get('verbose')
|
|
|
uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1)
|
|
|
exec = [
|
|
|
'valgrind',
|
|
|
@@ -548,7 +555,7 @@ class TestSuite:
|
|
|
mk.write('\n')
|
|
|
|
|
|
# add coverage hooks?
|
|
|
- if args.get('coverage', False):
|
|
|
+ if args.get('coverage'):
|
|
|
mk.write(COVERAGE_RULES.replace(4*' ', '\t') % dict(
|
|
|
path=self.path))
|
|
|
mk.write('\n')
|
|
|
@@ -593,7 +600,7 @@ class TestSuite:
|
|
|
if not args.get('verbose', True):
|
|
|
sys.stdout.write(FAIL)
|
|
|
sys.stdout.flush()
|
|
|
- if not args.get('keep_going', False):
|
|
|
+ if not args.get('keep_going'):
|
|
|
if not args.get('verbose', True):
|
|
|
sys.stdout.write('\n')
|
|
|
raise
|
|
|
@@ -615,11 +622,11 @@ def main(**args):
|
|
|
|
|
|
# and what class of TestCase to run
|
|
|
classes = []
|
|
|
- if args.get('normal', False):
|
|
|
+ if args.get('normal'):
|
|
|
classes.append(TestCase)
|
|
|
- if args.get('reentrant', False):
|
|
|
+ if args.get('reentrant'):
|
|
|
classes.append(ReentrantTestCase)
|
|
|
- if args.get('valgrind', False):
|
|
|
+ if args.get('valgrind'):
|
|
|
classes.append(ValgrindTestCase)
|
|
|
if not classes:
|
|
|
classes = [TestCase]
|
|
|
@@ -664,7 +671,7 @@ def main(**args):
|
|
|
list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
|
|
|
[target for target in targets])
|
|
|
mpty, spty = pty.openpty()
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
proc = sp.Popen(cmd, stdout=spty, stderr=spty)
|
|
|
os.close(spty)
|
|
|
@@ -678,14 +685,14 @@ def main(**args):
|
|
|
break
|
|
|
raise
|
|
|
stdout.append(line)
|
|
|
- if args.get('verbose', False):
|
|
|
+ if args.get('verbose'):
|
|
|
sys.stdout.write(line)
|
|
|
# intercept warnings
|
|
|
m = re.match(
|
|
|
'^{0}([^:]+):(\d+):(?:\d+:)?{0}{1}:{0}(.*)$'
|
|
|
.format('(?:\033\[[\d;]*.| )*', 'warning'),
|
|
|
line)
|
|
|
- if m and not args.get('verbose', False):
|
|
|
+ if m and not args.get('verbose'):
|
|
|
try:
|
|
|
with open(m.group(1)) as f:
|
|
|
lineno = int(m.group(2))
|
|
|
@@ -698,9 +705,8 @@ def main(**args):
|
|
|
except:
|
|
|
pass
|
|
|
proc.wait()
|
|
|
-
|
|
|
if proc.returncode != 0:
|
|
|
- if not args.get('verbose', False):
|
|
|
+ if not args.get('verbose'):
|
|
|
for line in stdout:
|
|
|
sys.stdout.write(line)
|
|
|
sys.exit(-1)
|
|
|
@@ -718,7 +724,7 @@ def main(**args):
|
|
|
print('filtered down to %d permutations' % total)
|
|
|
|
|
|
# only requested to build?
|
|
|
- if args.get('build', False):
|
|
|
+ if args.get('build'):
|
|
|
return 0
|
|
|
|
|
|
print('====== testing ======')
|
|
|
@@ -733,12 +739,9 @@ def main(**args):
|
|
|
failed = 0
|
|
|
for suite in suites:
|
|
|
for perm in suite.perms:
|
|
|
- if not hasattr(perm, 'result'):
|
|
|
- continue
|
|
|
-
|
|
|
if perm.result == PASS:
|
|
|
passed += 1
|
|
|
- else:
|
|
|
+ elif isinstance(perm.result, TestFailure):
|
|
|
sys.stdout.write(
|
|
|
"\033[01m{path}:{lineno}:\033[01;31mfailure:\033[m "
|
|
|
"{perm} failed\n".format(
|
|
|
@@ -759,25 +762,33 @@ def main(**args):
|
|
|
sys.stdout.write('\n')
|
|
|
failed += 1
|
|
|
|
|
|
- if args.get('coverage', False):
|
|
|
+ if args.get('coverage'):
|
|
|
# collect coverage info
|
|
|
- # why -j1? lcov doesn't work in parallel because of gcov issues
|
|
|
+ # why -j1? lcov doesn't work in parallel because of gcov limitations
|
|
|
cmd = (['make', '-j1', '-f', 'Makefile'] +
|
|
|
list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
|
|
|
- [re.sub('\.test$', '.cumul.info', target) for target in targets])
|
|
|
- if args.get('verbose', False):
|
|
|
+ (['COVERAGETARGET=%s' % args['coverage']]
|
|
|
+ if isinstance(args['coverage'], str) else []) +
|
|
|
+ [suite.path + '.info' for suite in suites
|
|
|
+ if any(perm.result == PASS for perm in suite.perms)])
|
|
|
+ if args.get('verbose'):
|
|
|
print(' '.join(shlex.quote(c) for c in cmd))
|
|
|
proc = sp.Popen(cmd,
|
|
|
- stdout=sp.DEVNULL if not args.get('verbose', False) else None)
|
|
|
+ stdout=sp.PIPE if not args.get('verbose') else None,
|
|
|
+ stderr=sp.STDOUT if not args.get('verbose') else None,
|
|
|
+ universal_newlines=True)
|
|
|
proc.wait()
|
|
|
if proc.returncode != 0:
|
|
|
+ if not args.get('verbose'):
|
|
|
+ for line in proc.stdout:
|
|
|
+ sys.stdout.write(line)
|
|
|
sys.exit(-1)
|
|
|
|
|
|
- if args.get('gdb', False):
|
|
|
+ if args.get('gdb'):
|
|
|
failure = None
|
|
|
for suite in suites:
|
|
|
for perm in suite.perms:
|
|
|
- if getattr(perm, 'result', PASS) != PASS:
|
|
|
+ if isinstance(perm.result, TestFailure):
|
|
|
failure = perm.result
|
|
|
if failure is not None:
|
|
|
print('======= gdb ======')
|
|
|
@@ -827,11 +838,11 @@ if __name__ == "__main__":
|
|
|
help="Run tests with another executable prefixed on the command line.")
|
|
|
parser.add_argument('--disk',
|
|
|
help="Specify a file to use for persistent/reentrant tests.")
|
|
|
- parser.add_argument('--coverage', action='store_true',
|
|
|
+ parser.add_argument('--coverage', type=lambda x: x if x else True,
|
|
|
+ nargs='?', const='',
|
|
|
help="Collect coverage information during testing. This uses lcov/gcov \
|
|
|
- to accumulate coverage information into *.info files. Note \
|
|
|
- coverage is not reset between runs, allowing multiple runs to \
|
|
|
- contribute to coverage.")
|
|
|
+ to accumulate coverage information into *.info files. May also \
|
|
|
+ a path to a *.info file to accumulate coverage info into.")
|
|
|
parser.add_argument('--build-dir',
|
|
|
help="Build relative to the specified directory instead of the \
|
|
|
current directory.")
|
Christopher Haster