Tweaked/fixed a number of small runner things after a bit of use

- Added support for negative numbers in the leb16 encoding with an
  optional 'w' prefix.

- Changed prettyasserts.py rule to .a.c => .c, allowing other .a.c files
  in the future.

- Updated .gitignore with missing generated files (tags, .csv).

- Removed suite-namespacing of test symbols, these are no longer needed.

- Changed test define overrides to have higher priority than explicit
  defines encoded in test ids. So:

    ./runners/bench_runner bench_dir_open:0f1g12gg2b8c8dgg4e0 -DREAD_SIZE=16

  Behaves as expected.

  Otherwise it's not easy to experiment with known failing test cases.

- Fixed issue where the -b flag ignored explicit test/bench ids.

.gitignore

@@ -4,19 +4,31 @@
 *.a
 *.ci
 *.csv
-*.t.c
-*.b.c
-*.a.c
+*.t.*
+*.b.*
 *.gcno
 *.gcda
 *.perf
+lfs
+liblfs.a
 
 # Testing things
-blocks/
-lfs
-test.c
-tests/*.toml.*
-scripts/__pycache__
-.gdb_history
 runners/test_runner
 runners/bench_runner
+lfs.code.csv
+lfs.data.csv
+lfs.stack.csv
+lfs.structs.csv
+lfs.cov.csv
+lfs.perf.csv
+lfs.perfbd.csv
+lfs.test.csv
+lfs.bench.csv
+
+# Misc
+tags
+.gdb_history
+scripts/__pycache__
+
+# Historical, probably should remove at some point
+tests/*.toml.*

Makefile

@@ -28,43 +28,43 @@ VALGRIND ?= valgrind
 GDB		 ?= gdb
 PERF	 ?= perf
 
-SRC  ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c))
+SRC  ?= $(filter-out $(wildcard *.t.* *.b.*),$(wildcard *.c))
 OBJ  := $(SRC:%.c=$(BUILDDIR)/%.o)
 DEP  := $(SRC:%.c=$(BUILDDIR)/%.d)
 ASM  := $(SRC:%.c=$(BUILDDIR)/%.s)
 CI   := $(SRC:%.c=$(BUILDDIR)/%.ci)
-GCDA := $(SRC:%.c=$(BUILDDIR)/%.t.a.gcda)
+GCDA := $(SRC:%.c=$(BUILDDIR)/%.t.gcda)
 
 TESTS ?= $(wildcard tests/*.toml)
 TEST_SRC ?= $(SRC) \
-		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
+		$(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \
 		runners/test_runner.c
 TEST_RUNNER ?= $(BUILDDIR)/runners/test_runner
-TEST_TC    := $(TESTS:%.toml=$(BUILDDIR)/%.t.c) \
-		$(TEST_SRC:%.c=$(BUILDDIR)/%.t.c)
-TEST_TAC   := $(TEST_TC:%.t.c=%.t.a.c)
-TEST_OBJ   := $(TEST_TAC:%.t.a.c=%.t.a.o)
-TEST_DEP   := $(TEST_TAC:%.t.a.c=%.t.a.d)
-TEST_CI	   := $(TEST_TAC:%.t.a.c=%.t.a.ci)
-TEST_GCNO  := $(TEST_TAC:%.t.a.c=%.t.a.gcno)
-TEST_GCDA  := $(TEST_TAC:%.t.a.c=%.t.a.gcda)
+TEST_A     := $(TESTS:%.toml=$(BUILDDIR)/%.t.a.c) \
+		$(TEST_SRC:%.c=$(BUILDDIR)/%.t.a.c)
+TEST_C     := $(TEST_A:%.t.a.c=%.t.c)
+TEST_OBJ   := $(TEST_C:%.t.c=%.t.o)
+TEST_DEP   := $(TEST_C:%.t.c=%.t.d)
+TEST_CI	   := $(TEST_C:%.t.c=%.t.ci)
+TEST_GCNO  := $(TEST_C:%.t.c=%.t.gcno)
+TEST_GCDA  := $(TEST_C:%.t.c=%.t.gcda)
 TEST_PERF  := $(TEST_RUNNER:%=%.perf)
 TEST_TRACE := $(TEST_RUNNER:%=%.trace)
 TEST_CSV   := $(TEST_RUNNER:%=%.csv)
 
 BENCHES ?= $(wildcard benches/*.toml)
 BENCH_SRC ?= $(SRC) \
-		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
+		$(filter-out $(wildcard bd/*.t.* bd/*.b.*),$(wildcard bd/*.c)) \
 		runners/bench_runner.c
 BENCH_RUNNER ?= $(BUILDDIR)/runners/bench_runner
-BENCH_BC    := $(BENCHES:%.toml=$(BUILDDIR)/%.b.c) \
-		$(BENCH_SRC:%.c=$(BUILDDIR)/%.b.c)
-BENCH_BAC   := $(BENCH_BC:%.b.c=%.b.a.c)
-BENCH_OBJ   := $(BENCH_BAC:%.b.a.c=%.b.a.o)
-BENCH_DEP   := $(BENCH_BAC:%.b.a.c=%.b.a.d)
-BENCH_CI    := $(BENCH_BAC:%.b.a.c=%.b.a.ci)
-BENCH_GCNO  := $(BENCH_BAC:%.b.a.c=%.b.a.gcno)
-BENCH_GCDA  := $(BENCH_BAC:%.b.a.c=%.b.a.gcda)
+BENCH_A     := $(BENCHES:%.toml=$(BUILDDIR)/%.b.a.c) \
+		$(BENCH_SRC:%.c=$(BUILDDIR)/%.b.a.c)
+BENCH_C     := $(BENCH_A:%.b.a.c=%.b.c)
+BENCH_OBJ   := $(BENCH_C:%.b.c=%.b.o)
+BENCH_DEP   := $(BENCH_C:%.b.c=%.b.d)
+BENCH_CI    := $(BENCH_C:%.b.c=%.b.ci)
+BENCH_GCNO  := $(BENCH_C:%.b.c=%.b.gcno)
+BENCH_GCDA  := $(BENCH_C:%.b.c=%.b.gcda)
 BENCH_PERF  := $(BENCH_RUNNER:%=%.perf)
 BENCH_TRACE := $(BENCH_RUNNER:%=%.trace)
 BENCH_CSV   := $(BENCH_RUNNER:%=%.csv)
@@ -517,22 +517,22 @@ $(BUILDDIR)/%.o $(BUILDDIR)/%.ci: %.c
 $(BUILDDIR)/%.s: %.c
 	$(CC) -S $(CFLAGS) $< -o $@
 
-$(BUILDDIR)/%.a.c: %.c
+$(BUILDDIR)/%.c: %.a.c
 	./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@
 
-$(BUILDDIR)/%.a.c: $(BUILDDIR)/%.c
+$(BUILDDIR)/%.c: $(BUILDDIR)/%.a.c
 	./scripts/prettyasserts.py -p LFS_ASSERT $< -o $@
 
-$(BUILDDIR)/%.t.c: %.toml
+$(BUILDDIR)/%.t.a.c: %.toml
 	./scripts/test.py -c $< $(TESTCFLAGS) -o $@
 
-$(BUILDDIR)/%.t.c: %.c $(TESTS)
+$(BUILDDIR)/%.t.a.c: %.c $(TESTS)
 	./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@
 
-$(BUILDDIR)/%.b.c: %.toml
+$(BUILDDIR)/%.b.a.c: %.toml
 	./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@
 
-$(BUILDDIR)/%.b.c: %.c $(BENCHES)
+$(BUILDDIR)/%.b.a.c: %.c $(BENCHES)
 	./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@
 
 ## Clean everything
@@ -554,8 +554,8 @@ clean:
 	rm -f $(ASM)
 	rm -f $(CI)
 	rm -f $(TEST_RUNNER)
-	rm -f $(TEST_TC)
-	rm -f $(TEST_TAC)
+	rm -f $(TEST_A)
+	rm -f $(TEST_C)
 	rm -f $(TEST_OBJ)
 	rm -f $(TEST_DEP)
 	rm -f $(TEST_CI)
@@ -565,8 +565,8 @@ clean:
 	rm -f $(TEST_TRACE)
 	rm -f $(TEST_CSV)
 	rm -f $(BENCH_RUNNER)
-	rm -f $(BENCH_BC)
-	rm -f $(BENCH_BAC)
+	rm -f $(BENCH_A)
+	rm -f $(BENCH_C)
 	rm -f $(BENCH_OBJ)
 	rm -f $(BENCH_DEP)
 	rm -f $(BENCH_CI)

runners/bench_runner.c

@@ -52,6 +52,12 @@ void *mappend(void **p,
 
 // a quick self-terminating text-safe varint scheme
 static void leb16_print(uintmax_t x) {
+    // allow 'w' to indicate negative numbers
+    if ((intmax_t)x < 0) {
+        printf("w");
+        x = -x;
+    }
+
     while (true) {
         char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
         printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
@@ -63,7 +69,17 @@ static void leb16_print(uintmax_t x) {
 }
 
 static uintmax_t leb16_parse(const char *s, char **tail) {
+    bool neg = false;
     uintmax_t x = 0;
+    if (tail) {
+        *tail = (char*)s;
+    }
+
+    if (s[0] == 'w') {
+        neg = true;
+        s = s+1;
+    }
+
     size_t i = 0;
     while (true) {
         uintmax_t nibble = s[i];
@@ -73,23 +89,21 @@ static uintmax_t leb16_parse(const char *s, char **tail) {
             nibble = nibble - 'a' + 10;
         } else {
             // invalid?
-            if (tail) {
-                *tail = (char*)s;
-            }
             return 0;
         }
 
         x |= (nibble & 0xf) << (4*i);
         i += 1;
         if (!(nibble & 0x10)) {
+            s = s + i;
             break;
         }
     }
 
     if (tail) {
-        *tail = (char*)s + i;
+        *tail = (char*)s;
     }
-    return x;
+    return neg ? -x : x;
 }
 
 
@@ -145,8 +159,8 @@ intmax_t bench_define_lit(void *data) {
     BENCH_IMPLICIT_DEFINES
 #undef BENCH_DEF
 
-#define BENCH_DEFINE_MAP_EXPLICIT    0
-#define BENCH_DEFINE_MAP_OVERRIDE    1
+#define BENCH_DEFINE_MAP_OVERRIDE    0
+#define BENCH_DEFINE_MAP_EXPLICIT    1
 #define BENCH_DEFINE_MAP_PERMUTATION 2
 #define BENCH_DEFINE_MAP_GEOMETRY    3
 #define BENCH_DEFINE_MAP_IMPLICIT    4
@@ -681,11 +695,18 @@ static void case_forperm(
             const struct bench_suite *suite,
             const struct bench_case *case_),
         void *data) {
+    // explicit permutation?
     if (defines) {
         bench_define_explicit(defines, define_count);
-        bench_define_flush();
 
-        cb(data, suite, case_);
+        for (size_t v = 0; v < bench_override_define_permutations; v++) {
+            // define override permutation
+            bench_define_override(v);
+            bench_define_flush();
+
+            cb(data, suite, case_);
+        }
+
         return;
     }
 

runners/test_runner.c

@@ -52,6 +52,12 @@ void *mappend(void **p,
 
 // a quick self-terminating text-safe varint scheme
 static void leb16_print(uintmax_t x) {
+    // allow 'w' to indicate negative numbers
+    if ((intmax_t)x < 0) {
+        printf("w");
+        x = -x;
+    }
+
     while (true) {
         char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
         printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
@@ -63,7 +69,17 @@ static void leb16_print(uintmax_t x) {
 }
 
 static uintmax_t leb16_parse(const char *s, char **tail) {
+    bool neg = false;
     uintmax_t x = 0;
+    if (tail) {
+        *tail = (char*)s;
+    }
+
+    if (s[0] == 'w') {
+        neg = true;
+        s = s+1;
+    }
+
     size_t i = 0;
     while (true) {
         uintmax_t nibble = s[i];
@@ -73,23 +89,21 @@ static uintmax_t leb16_parse(const char *s, char **tail) {
             nibble = nibble - 'a' + 10;
         } else {
             // invalid?
-            if (tail) {
-                *tail = (char*)s;
-            }
             return 0;
         }
 
         x |= (nibble & 0xf) << (4*i);
         i += 1;
         if (!(nibble & 0x10)) {
+            s = s + i;
             break;
         }
     }
 
     if (tail) {
-        *tail = (char*)s + i;
+        *tail = (char*)s;
     }
-    return x;
+    return neg ? -x : x;
 }
 
 
@@ -158,8 +172,8 @@ intmax_t test_define_lit(void *data) {
     TEST_IMPLICIT_DEFINES
 #undef TEST_DEF
 
-#define TEST_DEFINE_MAP_EXPLICIT    0
-#define TEST_DEFINE_MAP_OVERRIDE    1
+#define TEST_DEFINE_MAP_OVERRIDE    0
+#define TEST_DEFINE_MAP_EXPLICIT    1
 #define TEST_DEFINE_MAP_PERMUTATION 2
 #define TEST_DEFINE_MAP_GEOMETRY    3
 #define TEST_DEFINE_MAP_IMPLICIT    4
@@ -675,26 +689,34 @@ static void case_forperm(
             const struct test_case *case_,
             const test_powerloss_t *powerloss),
         void *data) {
+    // explicit permutation?
     if (defines) {
         test_define_explicit(defines, define_count);
-        test_define_flush();
 
-        if (cycles) {
-            cb(data, suite, case_, &(test_powerloss_t){
-                    .run=run_powerloss_cycles,
-                    .cycles=cycles,
-                    .cycle_count=cycle_count});
-        } else {
-            for (size_t p = 0; p < test_powerloss_count; p++) {
-                // skip non-reentrant tests when powerloss testing
-                if (test_powerlosses[p].run != run_powerloss_none
-                        && !(case_->flags & TEST_REENTRANT)) {
-                    continue;
-                }
+        for (size_t v = 0; v < test_override_define_permutations; v++) {
+            // define override permutation
+            test_define_override(v);
+            test_define_flush();
+
+            // explicit powerloss cycles?
+            if (cycles) {
+                cb(data, suite, case_, &(test_powerloss_t){
+                        .run=run_powerloss_cycles,
+                        .cycles=cycles,
+                        .cycle_count=cycle_count});
+            } else {
+                for (size_t p = 0; p < test_powerloss_count; p++) {
+                    // skip non-reentrant tests when powerloss testing
+                    if (test_powerlosses[p].run != run_powerloss_none
+                            && !(case_->flags & TEST_REENTRANT)) {
+                        continue;
+                    }
 
-                cb(data, suite, case_, &test_powerlosses[p]);
+                    cb(data, suite, case_, &test_powerlosses[p]);
+                }
             }
         }
+
         return;
     }
 

scripts/bench.py

@@ -315,8 +315,8 @@ def compile(bench_paths, **args):
                         for i, defines in enumerate(case.permutations):
                             for k, v in sorted(defines.items()):
                                 if v not in define_cbs:
-                                    name = ('__bench__%s__%s__%s__%d'
-                                        % (suite.name, case.name, k, i))
+                                    name = ('__bench__%s__%s__%d'
+                                        % (case.name, k, i))
                                     define_cbs[v] = name
                                     f.writeln('intmax_t %s('
                                         '__attribute__((unused)) '
@@ -325,9 +325,9 @@ def compile(bench_paths, **args):
                                     f.writeln('}')
                                     f.writeln()
                         f.writeln('const bench_define_t '
-                            '__bench__%s__%s__defines[]['
+                            '__bench__%s__defines[]['
                             'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {'
-                            % (suite.name, case.name, len(suite.defines)))
+                            % (case.name, len(suite.defines)))
                         for defines in case.permutations:
                             f.writeln(4*' '+'{')
                             for k, v in sorted(defines.items()):
@@ -339,8 +339,8 @@ def compile(bench_paths, **args):
 
                     # create case filter function
                     if suite.if_ is not None or case.if_ is not None:
-                        f.writeln('bool __bench__%s__%s__filter(void) {'
-                            % (suite.name, case.name))
+                        f.writeln('bool __bench__%s__filter(void) {'
+                            % (case.name))
                         f.writeln(4*' '+'return %s;'
                             % ' && '.join('(%s)' % if_
                                 for if_ in [suite.if_, case.if_]
@@ -349,9 +349,9 @@ def compile(bench_paths, **args):
                         f.writeln()
 
                     # create case run function
-                    f.writeln('void __bench__%s__%s__run('
+                    f.writeln('void __bench__%s__run('
                         '__attribute__((unused)) struct lfs_config *cfg) {'
-                        % (suite.name, case.name))
+                        % (case.name))
                     f.writeln(4*' '+'// bench case %s' % case.name)
                     if case.code_lineno is not None:
                         f.writeln(4*' '+'#line %d "%s"'
@@ -391,16 +391,16 @@ def compile(bench_paths, **args):
                     else:
                         if case.defines:
                             f.writeln('extern const bench_define_t '
-                                '__bench__%s__%s__defines[]['
+                                '__bench__%s__defines[]['
                                 'BENCH_IMPLICIT_DEFINE_COUNT+%d];'
-                                % (suite.name, case.name, len(suite.defines)))
+                                % (case.name, len(suite.defines)))
                         if suite.if_ is not None or case.if_ is not None:
-                            f.writeln('extern bool __bench__%s__%s__filter('
+                            f.writeln('extern bool __bench__%s__filter('
                                 'void);'
-                                % (suite.name, case.name))
-                        f.writeln('extern void __bench__%s__%s__run('
+                                % (case.name))
+                        f.writeln('extern void __bench__%s__run('
                             'struct lfs_config *cfg);'
-                            % (suite.name, case.name))
+                            % (case.name))
                         f.writeln()
 
                 # create suite struct
@@ -436,13 +436,13 @@ def compile(bench_paths, **args):
                         % len(case.permutations))
                     if case.defines:
                         f.writeln(12*' '+'.defines '
-                            '= (const bench_define_t*)__bench__%s__%s__defines,'
-                            % (suite.name, case.name))
+                            '= (const bench_define_t*)__bench__%s__defines,'
+                            % (case.name))
                     if suite.if_ is not None or case.if_ is not None:
-                        f.writeln(12*' '+'.filter = __bench__%s__%s__filter,'
-                            % (suite.name, case.name))
-                    f.writeln(12*' '+'.run = __bench__%s__%s__run,'
-                        % (suite.name, case.name))
+                        f.writeln(12*' '+'.filter = __bench__%s__filter,'
+                            % (case.name))
+                    f.writeln(12*' '+'.run = __bench__%s__run,'
+                        % (case.name))
                     f.writeln(8*' '+'},')
                 f.writeln(4*' '+'},')
                 f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
@@ -1040,7 +1040,8 @@ def run(runner, bench_ids=[], **args):
     proged = 0
     erased = 0
     failures = []
-    for by in (expected_case_perms.keys() if args.get('by_cases')
+    for by in (bench_ids if bench_ids
+            else expected_case_perms.keys() if args.get('by_cases')
             else expected_suite_perms.keys() if args.get('by_suites')
             else [None]):
         # spawn jobs for stage
@@ -1053,7 +1054,7 @@ def run(runner, bench_ids=[], **args):
             killed) = run_stage(
                 by or 'benches',
                 runner_,
-                [by] if by is not None else bench_ids,
+                [by] if by is not None else [],
                 stdout,
                 trace,
                 output,

scripts/test.py

@@ -323,8 +323,8 @@ def compile(test_paths, **args):
                         for i, defines in enumerate(case.permutations):
                             for k, v in sorted(defines.items()):
                                 if v not in define_cbs:
-                                    name = ('__test__%s__%s__%s__%d'
-                                        % (suite.name, case.name, k, i))
+                                    name = ('__test__%s__%s__%d'
+                                        % (case.name, k, i))
                                     define_cbs[v] = name
                                     f.writeln('intmax_t %s('
                                         '__attribute__((unused)) '
@@ -333,9 +333,9 @@ def compile(test_paths, **args):
                                     f.writeln('}')
                                     f.writeln()
                         f.writeln('const test_define_t '
-                            '__test__%s__%s__defines[]['
+                            '__test__%s__defines[]['
                             'TEST_IMPLICIT_DEFINE_COUNT+%d] = {'
-                            % (suite.name, case.name, len(suite.defines)))
+                            % (case.name, len(suite.defines)))
                         for defines in case.permutations:
                             f.writeln(4*' '+'{')
                             for k, v in sorted(defines.items()):
@@ -347,8 +347,8 @@ def compile(test_paths, **args):
 
                     # create case filter function
                     if suite.if_ is not None or case.if_ is not None:
-                        f.writeln('bool __test__%s__%s__filter(void) {'
-                            % (suite.name, case.name))
+                        f.writeln('bool __test__%s__filter(void) {'
+                            % (case.name))
                         f.writeln(4*' '+'return %s;'
                             % ' && '.join('(%s)' % if_
                                 for if_ in [suite.if_, case.if_]
@@ -357,9 +357,9 @@ def compile(test_paths, **args):
                         f.writeln()
 
                     # create case run function
-                    f.writeln('void __test__%s__%s__run('
+                    f.writeln('void __test__%s__run('
                         '__attribute__((unused)) struct lfs_config *cfg) {'
-                        % (suite.name, case.name))
+                        % (case.name))
                     f.writeln(4*' '+'// test case %s' % case.name)
                     if case.code_lineno is not None:
                         f.writeln(4*' '+'#line %d "%s"'
@@ -399,16 +399,16 @@ def compile(test_paths, **args):
                     else:
                         if case.defines:
                             f.writeln('extern const test_define_t '
-                                '__test__%s__%s__defines[]['
+                                '__test__%s__defines[]['
                                 'TEST_IMPLICIT_DEFINE_COUNT+%d];'
-                                % (suite.name, case.name, len(suite.defines)))
+                                % (case.name, len(suite.defines)))
                         if suite.if_ is not None or case.if_ is not None:
-                            f.writeln('extern bool __test__%s__%s__filter('
+                            f.writeln('extern bool __test__%s__filter('
                                 'void);'
-                                % (suite.name, case.name))
-                        f.writeln('extern void __test__%s__%s__run('
+                                % (case.name))
+                        f.writeln('extern void __test__%s__run('
                             'struct lfs_config *cfg);'
-                            % (suite.name, case.name))
+                            % (case.name))
                         f.writeln()
 
                 # create suite struct
@@ -450,13 +450,13 @@ def compile(test_paths, **args):
                         % len(case.permutations))
                     if case.defines:
                         f.writeln(12*' '+'.defines '
-                            '= (const test_define_t*)__test__%s__%s__defines,'
-                            % (suite.name, case.name))
+                            '= (const test_define_t*)__test__%s__defines,'
+                            % (case.name))
                     if suite.if_ is not None or case.if_ is not None:
-                        f.writeln(12*' '+'.filter = __test__%s__%s__filter,'
-                            % (suite.name, case.name))
-                    f.writeln(12*' '+'.run = __test__%s__%s__run,'
-                        % (suite.name, case.name))
+                        f.writeln(12*' '+'.filter = __test__%s__filter,'
+                            % (case.name))
+                    f.writeln(12*' '+'.run = __test__%s__run,'
+                        % (case.name))
                     f.writeln(8*' '+'},')
                 f.writeln(4*' '+'},')
                 f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
@@ -1044,7 +1044,8 @@ def run(runner, test_ids=[], **args):
     passed = 0
     powerlosses = 0
     failures = []
-    for by in (expected_case_perms.keys() if args.get('by_cases')
+    for by in (test_ids if test_ids
+            else expected_case_perms.keys() if args.get('by_cases')
             else expected_suite_perms.keys() if args.get('by_suites')
             else [None]):
         # spawn jobs for stage
@@ -1055,7 +1056,7 @@ def run(runner, test_ids=[], **args):
             killed) = run_stage(
                 by or 'tests',
                 runner_,
-                [by] if by is not None else test_ids,
+                [by] if by is not None else [],
                 stdout,
                 trace,
                 output,