Reworked how multi-layered defines work in the test-runner

In the test-runner, defines are parameterized constants (limited
to integers) that are generated from the test suite tomls resulting
in many permutations of each test.

In order to make this efficient, these defines are implemented as
multi-layered lookup tables, using per-layer/per-scope indirect
mappings. This lets the test-runner and test suites define their
own defines with compile-time indexes independently. It also makes
building of the lookup tables very efficient, since they can be
incrementally populated as we expand the test permutations.

The four current define layers and when we need to build them:

layer                           defines         predefine_map   define_map
user-provided overrides         per-run         per-run         per-suite
per-permutation defines         per-perm        per-case        per-perm
per-geometry defines            per-perm        compile-time    -
default defines                 compile-time    compile-time    -

.gitignore

@@ -14,3 +14,4 @@ test.c
 tests/*.toml.*
 scripts/__pycache__
 .gdb_history
+runners/test_runner

Makefile

@@ -54,6 +54,9 @@ override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
 override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
 
+override TESTFLAGS_ += -b
+# forward -j flag
+override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS))
 ifdef VERBOSE
 override TESTFLAGS     += -v
 override CALLSFLAGS    += -v
@@ -88,8 +91,6 @@ endif
 ifneq ($(OBJDUMP),objdump)
 override STRUCTSFLAGS += --objdump-tool="$(OBJDUMP)"
 endif
-# forward -j flag
-override TESTFLAGS_ += $(filter -j%,$(MAKEFLAGS))
 
 
 # commands

runners/test_runner.c

@@ -6,106 +6,151 @@
 #include <sys/types.h>
 #include <errno.h>
 
-// disk geometries
+
+// test geometries
 struct test_geometry {
     const char *name;
-    const test_define_t *defines;
+    test_define_t defines[TEST_GEOMETRY_DEFINE_COUNT];
+};
+
+const struct test_geometry test_geometries[TEST_GEOMETRY_COUNT]
+        = TEST_GEOMETRIES;
+
+// test define lookup and management
+#define TEST_DEFINE_LAYERS 4
+const test_define_t *test_defines[TEST_DEFINE_LAYERS] = {
+    NULL,
+    NULL,
+    NULL,
+    (const test_define_t[TEST_DEFAULT_COUNT])TEST_DEFAULTS,
 };
 
-// Note this includes the default configuration for test pre-defines
-#define TEST_GEOMETRY(name, read, prog, erase, count) \
-    {name, (const test_define_t[]){ \
-        /* READ_SIZE */         read, \
-        /* PROG_SIZE */         prog, \
-        /* BLOCK_SIZE */        erase, \
-        /* BLOCK_COUNT */       count, \
-        /* BLOCK_CYCLES */      -1, \
-        /* CACHE_SIZE */        (64 % (prog) == 0) ? 64 : (prog), \
-        /* LOOKAHEAD_SIZE */    16, \
-        /* ERASE_VALUE */       0xff, \
-        /* ERASE_CYCLES */      0, \
-        /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
-    }}
-
-const struct test_geometry test_geometries[] = {
-    // Made up geometry that works well for testing
-    TEST_GEOMETRY("test",     16,   16,     512, (1024*1024)/512),
-    // EEPROM/NVRAM
-    TEST_GEOMETRY("eeprom",    1,    1,     512, (1024*1024)/512),
-    // SD/eMMC
-    TEST_GEOMETRY("emmc",    512,  512,     512, (1024*1024)/512),
-    // NOR flash                       
-    TEST_GEOMETRY("nor",       1,    1,    4096, (1024*1024)/4096),
-    // NAND flash                      
-    TEST_GEOMETRY("nand",   4096, 4096, 32*1024, (1024*1024)/(32*1024)),
+const uint8_t *test_predefine_maps[TEST_DEFINE_LAYERS] = {
+    NULL,
+    NULL,
+    (const uint8_t[TEST_PREDEFINE_COUNT])TEST_GEOMETRY_DEFINE_MAP,
+    (const uint8_t[TEST_PREDEFINE_COUNT])TEST_DEFAULT_MAP,
 };
 
-const size_t test_geometry_count = (
-        sizeof(test_geometries) / sizeof(test_geometries[0]));
+const uint8_t *test_define_maps[TEST_DEFINE_LAYERS] = {
+    NULL,
+    NULL,
+    NULL,
+    NULL,
+};
 
+uint8_t test_override_predefine_map[TEST_PREDEFINE_COUNT];
+uint8_t test_override_define_map[256];
+uint8_t test_case_predefine_map[TEST_PREDEFINE_COUNT];
 
-// test define lookup and management
-const test_define_t *test_defines[3] = {NULL};
-const uint8_t *test_define_maps[2] = {NULL};
+const char *const *test_override_names;
+size_t test_override_count;
 
-test_define_t test_define(size_t define) {
-    if (test_define_maps[0] && test_define_maps[0][define] != 0xff) {
-        return test_defines[0][test_define_maps[0][define]];
-    } else if (test_define_maps[1] && test_define_maps[1][define] != 0xff) {
-        return test_defines[1][test_define_maps[1][define]];
-    } else {
-        return test_defines[2][define];
+const char *const test_predefine_names[TEST_PREDEFINE_COUNT]
+        = TEST_PREDEFINE_NAMES;
+
+const char *const *test_define_names;
+size_t test_define_count;
+
+
+test_define_t test_predefine(size_t define) {
+    for (int i = 0; i < TEST_DEFINE_LAYERS; i++) {
+        if (test_defines[i]
+                && test_predefine_maps[i]
+                && test_predefine_maps[i][define] != 0xff) {
+            return test_defines[i][test_predefine_maps[i][define]];
+        }
     }
+
+    fprintf(stderr, "error: undefined predefine %s\n",
+            test_predefine_names[define]);
+    assert(false);
+    exit(-1);
 }
 
-static void test_define_geometry(const struct test_geometry *geometry) {
-    if (geometry) {
-        test_defines[2] = geometry->defines;
-    } else {
-        test_defines[2] = NULL;
+test_define_t test_define(size_t define) {
+    for (int i = 0; i < TEST_DEFINE_LAYERS; i++) {
+        if (test_defines[i]
+                && test_define_maps[i]
+                && test_define_maps[i][define] != 0xff) {
+            return test_defines[i][test_define_maps[i][define]];
+        }
     }
+
+    fprintf(stderr, "error: undefined define %s\n",
+            test_define_names[define]);
+    assert(false);
+    exit(-1);
 }
 
-static void test_define_case(const struct test_case *case_, size_t perm) {
-    if (case_ && case_->defines) {
-        test_defines[1] = case_->defines[perm];
-        test_define_maps[1] = case_->define_map;
-    } else {
-        test_defines[1] = NULL;
-        test_define_maps[1] = NULL;
-    }
+static void test_define_geometry(const struct test_geometry *geometry) {
+    test_defines[2] = geometry->defines;
 }
 
 static void test_define_overrides(
-        const struct test_suite *suite,
         const char *const *override_names,
         const test_define_t *override_defines,
         size_t override_count) {
-    if (override_names && override_defines && override_count > 0) {
-        uint8_t *define_map = malloc(suite->define_count * sizeof(uint8_t));
-        memset(define_map, 0xff, suite->define_count * sizeof(bool));
-
-        // lookup each override in the suite defines, they may have a
-        // different index in each suite
-        for (size_t i = 0; i < override_count; i++) {
-            size_t j = 0;
-            for (; j < suite->define_count; j++) {
-                if (strcmp(override_names[i], suite->define_names[j]) == 0) {
-                    break;
-                }
+    test_defines[0] = override_defines;
+    test_override_names = override_names;
+    test_override_count = override_count;
+
+    // map any predefines
+    memset(test_override_predefine_map, 0xff, TEST_PREDEFINE_COUNT);
+    for (size_t i = 0; i < override_count; i++) {
+        for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) {
+            if (strcmp(override_names[i], test_predefine_names[j]) == 0) {
+                test_override_predefine_map[j] = i;
             }
+        }
+    }
+    test_predefine_maps[0] = test_override_predefine_map;
+}
+
+static void test_define_suite(const struct test_suite *suite) {
+    test_define_names = suite->define_names;
+    test_define_count = suite->define_count;
 
-            if (j < suite->define_count) {
-                define_map[j] = i;
+    // map any defines
+    memset(test_override_define_map, 0xff, suite->define_count);
+    for (size_t i = 0; i < test_override_count; i++) {
+        for (size_t j = 0; j < suite->define_count; j++) {
+            if (strcmp(test_override_names[i], suite->define_names[j]) == 0) {
+                test_override_define_map[j] = i;
             }
         }
+    }
+    test_define_maps[0] = test_override_define_map;
+}
+
+static void test_define_case(
+        const struct test_suite *suite,
+        const struct test_case *case_) {
+    (void)suite;
+    // case_->define_map is already correct, but we need to do
+    // some fixup for the predefine map
+    test_define_maps[1] = case_->define_map;
+
+    memset(test_case_predefine_map, 0xff, TEST_PREDEFINE_COUNT);
+    for (size_t i = 0; i < test_define_count; i++) {
+        for (size_t j = 0; j < TEST_PREDEFINE_COUNT; j++) {
+            if (strcmp(test_define_names[i], test_predefine_names[j]) == 0) {
+                test_case_predefine_map[j] = case_->define_map[i];
+            }
+        }
+    }
+    test_predefine_maps[1] = test_case_predefine_map;
+}
 
-        test_defines[0] = override_defines;
-        test_define_maps[0] = define_map;
+static void test_define_perm(
+        const struct test_suite *suite,
+        const struct test_case *case_,
+        size_t perm) {
+    (void)suite;
+    if (case_->defines) {
+        test_defines[1] = case_->defines[perm];
     } else {
-        test_defines[0] = NULL;
-        free((uint8_t *)test_define_maps[0]);
-        test_define_maps[0] = NULL;
+        test_defines[1] = NULL;
     }
 }
 
@@ -120,11 +165,6 @@ static size_t test_skip = 0;
 static size_t test_count = -1;
 static size_t test_every = 1;
 
-static const char **override_names = NULL;
-static test_define_t *override_defines = NULL;
-static size_t override_count = 0;
-static size_t override_cap = 0;
-
 static const char *test_persist = NULL;
 FILE *test_trace = NULL;
 
@@ -140,7 +180,7 @@ static bool test_case_skip(const struct test_case *case_) {
 }
 
 static bool test_perm_skip(size_t perm) {
-    size_t geom_perm = perm % test_geometry_count;
+    size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
     return (test_perm != (size_t)-1 && perm != test_perm)
             || (test_geometry && (strcmp(
                 test_geometries[geom_perm].name,
@@ -153,7 +193,8 @@ static bool test_step_skip(size_t step) {
             && (step-test_skip) % test_every == 0);
 }
 
-static void test_case_sumpermutations(
+static void test_case_permcount(
+        const struct test_suite *suite,
         const struct test_case *case_,
         size_t *perms,
         size_t *filtered) {
@@ -161,7 +202,7 @@ static void test_case_sumpermutations(
     size_t filtered_ = 0;
 
     for (size_t perm = 0;
-            perm < test_geometry_count
+            perm < TEST_GEOMETRY_COUNT
                 * case_->permutations;
             perm++) {
         if (test_perm_skip(perm)) {
@@ -171,23 +212,18 @@ static void test_case_sumpermutations(
         perms_ += 1;
 
         // setup defines
-        size_t case_perm = perm / test_geometry_count;
-        size_t geom_perm = perm % test_geometry_count;
+        size_t case_perm = perm / TEST_GEOMETRY_COUNT;
+        size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
+        test_define_perm(suite, case_, case_perm);
         test_define_geometry(&test_geometries[geom_perm]);
-        test_define_case(case_, case_perm);
 
         if (case_->filter) {
             if (!case_->filter(case_perm)) {
-                test_define_geometry(NULL);
-                test_define_case(NULL, 0);
                 continue;
             }
         }
 
         filtered_ += 1;
-
-        test_define_geometry(NULL);
-        test_define_case(NULL, 0);
     }
 
     *perms += perms_;
@@ -208,21 +244,18 @@ static void summary(void) {
             continue;
         }
 
-        test_define_overrides(
-                test_suites[i],
-                override_names, override_defines, override_count);
+        test_define_suite(test_suites[i]);
 
         for (size_t j = 0; j < test_suites[i]->case_count; j++) {
             if (test_case_skip(test_suites[i]->cases[j])) {
                 continue;
             }
 
-            test_case_sumpermutations(test_suites[i]->cases[j],
+            test_define_case(test_suites[i], test_suites[i]->cases[j]);
+            test_case_permcount(test_suites[i], test_suites[i]->cases[j],
                     &perms, &filtered);
         }
 
-        test_define_overrides(NULL, NULL, NULL, 0);
-
         cases += test_suites[i]->case_count;
         types |= test_suites[i]->types;
     }
@@ -243,16 +276,13 @@ static void summary(void) {
 }
 
 static void list_suites(void) {
-    printf("%-36s %-12s %7s %7s %11s\n",
-            "id", "suite", "types", "cases", "perms");
+    printf("%-36s %7s %7s %11s\n", "suite", "types", "cases", "perms");
     for (size_t i = 0; i < test_suite_count; i++) {
         if (test_suite_skip(test_suites[i])) {
             continue;
         }
 
-        test_define_overrides(
-                test_suites[i],
-                override_names, override_defines, override_count);
+        test_define_suite(test_suites[i]);
 
         size_t perms = 0;
         size_t filtered = 0;
@@ -261,12 +291,11 @@ static void list_suites(void) {
                 continue;
             }
 
-            test_case_sumpermutations(test_suites[i]->cases[j],
+            test_define_case(test_suites[i], test_suites[i]->cases[j]);
+            test_case_permcount(test_suites[i], test_suites[i]->cases[j],
                     &perms, &filtered);
         }
 
-        test_define_overrides(NULL, NULL, NULL, 0);
-
         char perm_buf[64];
         sprintf(perm_buf, "%zu/%zu", filtered, perms);
         char type_buf[64];
@@ -274,9 +303,8 @@ static void list_suites(void) {
                 (test_suites[i]->types & TEST_NORMAL)    ? "n" : "",
                 (test_suites[i]->types & TEST_REENTRANT) ? "r" : "",
                 (test_suites[i]->types & TEST_VALGRIND)  ? "V" : "");
-        printf("%-36s %-12s %7s %7zu %11s\n",
+        printf("%-36s %7s %7zu %11s\n",
                 test_suites[i]->id,
-                test_suites[i]->name,
                 type_buf,
                 test_suites[i]->case_count,
                 perm_buf);
@@ -284,25 +312,24 @@ static void list_suites(void) {
 }
 
 static void list_cases(void) {
-    printf("%-36s %-12s %-12s %7s %11s\n",
-            "id", "suite", "case", "types", "perms");
+    printf("%-36s %7s %11s\n", "case", "types", "perms");
     for (size_t i = 0; i < test_suite_count; i++) {
         if (test_suite_skip(test_suites[i])) {
             continue;
         }
 
-        test_define_overrides(
-                test_suites[i],
-                override_names, override_defines, override_count);
+        test_define_suite(test_suites[i]);
 
         for (size_t j = 0; j < test_suites[i]->case_count; j++) {
             if (test_case_skip(test_suites[i]->cases[j])) {
                 continue;
             }
 
+            test_define_case(test_suites[i], test_suites[i]->cases[j]);
+
             size_t perms = 0;
             size_t filtered = 0;
-            test_case_sumpermutations(test_suites[i]->cases[j],
+            test_case_permcount(test_suites[i], test_suites[i]->cases[j],
                     &perms, &filtered);
             test_types_t types = test_suites[i]->cases[j]->types;
 
@@ -313,20 +340,15 @@ static void list_cases(void) {
                     (types & TEST_NORMAL)    ? "n" : "",
                     (types & TEST_REENTRANT) ? "r" : "",
                     (types & TEST_VALGRIND)  ? "V" : "");
-            printf("%-36s %-12s %-12s %7s %11s\n",
+            printf("%-36s %7s %11s\n",
                     test_suites[i]->cases[j]->id,
-                    test_suites[i]->name,
-                    test_suites[i]->cases[j]->name,
                     type_buf,
                     perm_buf);
         }
-
-        test_define_overrides(NULL, NULL, NULL, 0);
     }
 }
 
 static void list_paths(void) {
-    printf("%-36s %-36s\n", "id", "path");
     for (size_t i = 0; i < test_suite_count; i++) {
         if (test_suite_skip(test_suites[i])) {
             continue;
@@ -345,23 +367,22 @@ static void list_paths(void) {
 }
 
 static void list_defines(void) {
-    printf("%-36s %s\n", "id", "defines");
     for (size_t i = 0; i < test_suite_count; i++) {
         if (test_suite_skip(test_suites[i])) {
             continue;
         }
 
-        test_define_overrides(
-                test_suites[i],
-                override_names, override_defines, override_count);
+        test_define_suite(test_suites[i]);
 
         for (size_t j = 0; j < test_suites[i]->case_count; j++) {
             if (test_case_skip(test_suites[i]->cases[j])) {
                 continue;
             }
 
+            test_define_case(test_suites[i], test_suites[i]->cases[j]);
+
             for (size_t perm = 0;
-                    perm < test_geometry_count
+                    perm < TEST_GEOMETRY_COUNT
                         * test_suites[i]->cases[j]->permutations;
                     perm++) {
                 if (test_perm_skip(perm)) {
@@ -369,39 +390,38 @@ static void list_defines(void) {
                 }
 
                 // setup defines
-                size_t case_perm = perm / test_geometry_count;
-                size_t geom_perm = perm % test_geometry_count;
+                size_t case_perm = perm / TEST_GEOMETRY_COUNT;
+                size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
+                test_define_perm(test_suites[i],
+                        test_suites[i]->cases[j], case_perm);
                 test_define_geometry(&test_geometries[geom_perm]);
-                test_define_case(test_suites[i]->cases[j], case_perm);
 
-                // print each define
+                // print the case
                 char id_buf[256];
                 sprintf(id_buf, "%s#%zu", test_suites[i]->cases[j]->id, perm);
                 printf("%-36s ", id_buf);
+
+                // special case for the current geometry
+                printf("GEOMETRY=%s ", test_geometries[geom_perm].name);
+
+                // print each define
                 for (size_t k = 0; k < test_suites[i]->define_count; k++) {
-                    if (k >= TEST_PREDEFINE_COUNT && (
-                            !test_suites[i]->cases[j]->define_map
-                            || test_suites[i]->cases[j]->define_map[k]
-                                == 0xff)) {
-                        continue;
+                    if (test_suites[i]->cases[j]->define_map
+                            && test_suites[i]->cases[j]->define_map[k]
+                                != 0xff) {
+                        printf("%s=%jd ",
+                                test_suites[i]->define_names[k],
+                                test_define(k));
                     }
-
-                    printf("%s=%jd ",
-                            test_suites[i]->define_names[k],
-                            test_define(k));
                 }
                 printf("\n");
             }
         }
-
-        test_define_overrides(NULL, NULL, NULL, 0);
     }
 }
 
 static void list_geometries(void) {
-    printf("%-36s %7s %7s %7s %7s %7s\n",
-            "name", "read", "prog", "erase", "count", "size");
-    for (size_t i = 0; i < test_geometry_count; i++) {
+    for (size_t i = 0; i < TEST_GEOMETRY_COUNT; i++) {
         if (test_geometry && strcmp(
                 test_geometries[i].name,
                 test_geometry) != 0) {
@@ -410,16 +430,33 @@ static void list_geometries(void) {
 
         test_define_geometry(&test_geometries[i]);
 
-        printf("%-36s %7ju %7ju %7ju %7ju %7ju\n",
-                test_geometries[i].name,
-                READ_SIZE,
-                PROG_SIZE,
-                BLOCK_SIZE,
-                BLOCK_COUNT,
-                BLOCK_SIZE*BLOCK_COUNT);
+        printf("%-36s ", test_geometries[i].name);
+        // print each define
+        for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) {
+            if (test_predefine_maps[2][k] != 0xff) {
+                printf("%s=%jd ",
+                        test_predefine_names[k],
+                        test_predefine(k));
+            }
+        }
+        printf("\n");
+
     }
 }
 
+static void list_defaults(void) {
+    printf("%-36s ", "defaults");
+    // print each define
+    for (size_t k = 0; k < TEST_PREDEFINE_COUNT; k++) {
+        if (test_predefine_maps[3][k] != 0xff) {
+            printf("%s=%jd ",
+                    test_predefine_names[k],
+                    test_predefine(k));
+        }
+    }
+    printf("\n");
+}
+
 static void run(void) {
     size_t step = 0;
     for (size_t i = 0; i < test_suite_count; i++) {
@@ -427,23 +464,22 @@ static void run(void) {
             continue;
         }
 
-        test_define_overrides(
-                test_suites[i],
-                override_names, override_defines, override_count);
+        test_define_suite(test_suites[i]);
 
         for (size_t j = 0; j < test_suites[i]->case_count; j++) {
             if (test_case_skip(test_suites[i]->cases[j])) {
                 continue;
             }
 
+            test_define_case(test_suites[i], test_suites[i]->cases[j]);
+
             for (size_t perm = 0;
-                    perm < test_geometry_count
+                    perm < TEST_GEOMETRY_COUNT
                         * test_suites[i]->cases[j]->permutations;
                     perm++) {
                 if (test_perm_skip(perm)) {
                     continue;
                 }
-
                 if (test_step_skip(step)) {
                     step += 1;
                     continue;
@@ -451,10 +487,11 @@ static void run(void) {
                 step += 1;
 
                 // setup defines
-                size_t case_perm = perm / test_geometry_count;
-                size_t geom_perm = perm % test_geometry_count;
+                size_t case_perm = perm / TEST_GEOMETRY_COUNT;
+                size_t geom_perm = perm % TEST_GEOMETRY_COUNT;
+                test_define_perm(test_suites[i],
+                        test_suites[i]->cases[j], case_perm);
                 test_define_geometry(&test_geometries[geom_perm]);
-                test_define_case(test_suites[i]->cases[j], case_perm);
 
                 // filter?
                 if (test_suites[i]->cases[j]->filter) {
@@ -462,8 +499,6 @@ static void run(void) {
                         printf("skipped %s#%zu\n",
                                 test_suites[i]->cases[j]->id,
                                 perm);
-                        test_define_geometry(NULL);
-                        test_define_case(NULL, 0);
                         continue;
                     }
                 }
@@ -514,13 +549,8 @@ static void run(void) {
                             "could not destroy block device: %d\n", err);
                     exit(-1);
                 }
-
-                test_define_geometry(NULL);
-                test_define_case(NULL, 0);
             }
         }
-
-        test_define_overrides(NULL, NULL, NULL, 0);
     }
 }
 
@@ -536,14 +566,15 @@ enum opt_flags {
     OPT_LIST_PATHS      = 1,
     OPT_LIST_DEFINES    = 2,
     OPT_LIST_GEOMETRIES = 3,
+    OPT_LIST_DEFAULTS   = 4,
     OPT_DEFINE          = 'D',
     OPT_GEOMETRY        = 'G',
     OPT_NORMAL          = 'n',
     OPT_REENTRANT       = 'r',
     OPT_VALGRIND        = 'V',
-    OPT_SKIP            = 4,
-    OPT_COUNT           = 5,
-    OPT_EVERY           = 6,
+    OPT_SKIP            = 5,
+    OPT_COUNT           = 6,
+    OPT_EVERY           = 7,
     OPT_PERSIST         = 'p',
     OPT_TRACE           = 't',
 };
@@ -558,6 +589,7 @@ const struct option long_opts[] = {
     {"list-paths",      no_argument,       NULL, OPT_LIST_PATHS},
     {"list-defines",    no_argument,       NULL, OPT_LIST_DEFINES},
     {"list-geometries", no_argument,       NULL, OPT_LIST_GEOMETRIES},
+    {"list-defaults",   no_argument,       NULL, OPT_LIST_DEFAULTS},
     {"define",          required_argument, NULL, OPT_DEFINE},
     {"geometry",        required_argument, NULL, OPT_GEOMETRY},
     {"normal",          no_argument,       NULL, OPT_NORMAL},
@@ -579,6 +611,7 @@ const char *const help_text[] = {
     "List the path for each test case.",
     "List the defines for each test permutation.",
     "List the disk geometries used for testing.",
+    "List the default defines in this test-runner.",
     "Override a test define.",
     "Filter by geometry.",
     "Filter for normal tests. Can be combined.",
@@ -594,6 +627,11 @@ const char *const help_text[] = {
 int main(int argc, char **argv) {
     void (*op)(void) = run;
 
+    static const char **override_names = NULL;
+    static test_define_t *override_defines = NULL;
+    static size_t override_count = 0;
+    static size_t override_cap = 0;
+
     // parse options
     while (true) {
         int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
@@ -673,8 +711,18 @@ int main(int argc, char **argv) {
             case OPT_LIST_GEOMETRIES:
                 op = list_geometries;
                 break;
+            case OPT_LIST_DEFAULTS:
+                op = list_defaults;
+                break;
             // configuration
             case OPT_DEFINE: {
+                // special case for -DGEOMETRY=<name>, we treat this the same
+                // as --geometry=<name>
+                if (strncmp(optarg, "GEOMETRY=", strlen("GEOMETRY=")) == 0) {
+                    test_geometry = &optarg[strlen("GEOMETRY=")];
+                    break;
+                }
+
                 // realloc if necessary
                 override_count += 1;
                 if (override_count > override_cap) {
@@ -816,6 +864,9 @@ getopt_done: ;
         test_suite = suite;
     }
 
+    // register overrides
+    test_define_overrides(override_names, override_defines, override_count);
+
     // do the thing
     op();
 

runners/test_runner.h

@@ -41,27 +41,68 @@ struct test_suite {
     size_t case_count;
 };
 
-// TODO remove this indirection
 extern const struct test_suite *test_suites[];
 extern const size_t test_suite_count;
 
 
 // access generated test defines
+test_define_t test_predefine(size_t define);
 test_define_t test_define(size_t define);
 
 // a few preconfigured defines that control how tests run
-#define READ_SIZE           test_define(0)
-#define PROG_SIZE           test_define(1)
-#define BLOCK_SIZE          test_define(2)
-#define BLOCK_COUNT         test_define(3)
-#define BLOCK_CYCLES        test_define(4)
-#define CACHE_SIZE          test_define(5)
-#define LOOKAHEAD_SIZE      test_define(6)
-#define ERASE_VALUE         test_define(7)
-#define ERASE_CYCLES        test_define(8)
-#define BADBLOCK_BEHAVIOR   test_define(9)
-
+#define READ_SIZE           test_predefine(0)
+#define PROG_SIZE           test_predefine(1)
+#define BLOCK_SIZE          test_predefine(2)
+#define BLOCK_COUNT         test_predefine(3)
+#define CACHE_SIZE          test_predefine(4)
+#define LOOKAHEAD_SIZE      test_predefine(5)
+#define BLOCK_CYCLES        test_predefine(6)
+#define ERASE_VALUE         test_predefine(7)
+#define ERASE_CYCLES        test_predefine(8)
+#define BADBLOCK_BEHAVIOR   test_predefine(9)
+
+#define TEST_PREDEFINE_NAMES { \
+    "READ_SIZE", \
+    "PROG_SIZE", \
+    "BLOCK_SIZE", \
+    "BLOCK_COUNT", \
+    "CACHE_SIZE", \
+    "LOOKAHEAD_SIZE", \
+    "BLOCK_CYCLES", \
+    "ERASE_VALUE", \
+    "ERASE_CYCLES", \
+    "BADBLOCK_BEHAVIOR", \
+}
 #define TEST_PREDEFINE_COUNT 10
 
 
+// default predefines
+#define TEST_DEFAULTS { \
+    /* LOOKAHEAD_SIZE */    16, \
+    /* BLOCK_CYCLES */      -1, \
+    /* ERASE_VALUE */       0xff, \
+    /* ERASE_CYCLES */      0, \
+    /* BADBLOCK_BEHAVIOR */ LFS_TESTBD_BADBLOCK_PROGERROR, \
+}
+#define TEST_DEFAULT_COUNT 5
+#define TEST_DEFAULT_MAP { \
+    0xff, 0xff, 0xff, 0xff, 0xff, 0, 1, 2, 3, 4 \
+}
+
+// test geometries
+#define TEST_GEOMETRIES { \
+    /*geometry, read, write,    erase,                 count, cache */ \
+    {"test",   {  16,    16,      512,       (1024*1024)/512,    64}}, \
+    {"eeprom", {   1,     1,      512,       (1024*1024)/512,    64}}, \
+    {"emmc",   { 512,   512,      512,       (1024*1024)/512,   512}}, \
+    {"nor",    {   1,     1,     4096,      (1024*1024)/4096,    64}}, \
+    {"nand",   {4096,  4096,  32*1024, (1024*1024)/(32*1024),  4096}}, \
+}
+#define TEST_GEOMETRY_COUNT 5
+#define TEST_GEOMETRY_DEFINE_COUNT 5
+#define TEST_GEOMETRY_DEFINE_MAP { \
+    0, 1, 2, 3, 4, 0xff, 0xff, 0xff, 0xff, 0xff \
+}
+
+
 #endif

scripts/test_.py

@@ -31,30 +31,21 @@ CASE_PROLOGUE = """
 CASE_EPILOGUE = """
 """
 
-TEST_PREDEFINES = [
-    'READ_SIZE',
-    'PROG_SIZE',
-    'BLOCK_SIZE',
-    'BLOCK_COUNT',
-    'BLOCK_CYCLES',
-    'CACHE_SIZE',
-    'LOOKAHEAD_SIZE',
-    'ERASE_VALUE',
-    'ERASE_CYCLES',
-    'BADBLOCK_BEHAVIOR',
-]
-
-
-# TODO
-# def testpath(path):
-# def testcase(path):
-# def testperm(path):
+
+def testpath(path):
+    path, *_ = path.split('#', 1)
+    return path
 
 def testsuite(path):
-    name = os.path.basename(path)
-    if name.endswith('.toml'):
-        name = name[:-len('.toml')]
-    return name
+    suite = testpath(path)
+    suite = os.path.basename(suite)
+    if suite.endswith('.toml'):
+        suite = suite[:-len('.toml')]
+    return suite
+
+def testcase(path):
+    _, case, *_ = path.split('#', 2)
+    return '%s#%s' % (testsuite(path), case)
 
 # TODO move this out in other files
 def openio(path, mode='r'):
@@ -111,7 +102,7 @@ class TestSuite:
     # create a TestSuite object from a toml file
     def __init__(self, path, args={}):
         self.name = testsuite(path)
-        self.path = path
+        self.path = testpath(path)
 
         # load toml file and parse test cases
         with open(self.path) as f:
@@ -125,9 +116,9 @@ class TestSuite:
             code_linenos = []
             for i, line in enumerate(f):
                 match = re.match(
-                    '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])' +
-                    '|(?P<if>if\s*=)'
-                    '|(?P<code>code\s*=)',
+                    '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
+                        '|' '(?P<if>if\s*=)'
+                        '|' '(?P<code>code\s*=)',
                     line)
                 if match and match.group('case'):
                     case_linenos.append((i+1, match.group('name')))
@@ -187,8 +178,8 @@ class TestSuite:
                     'suite_valgrind': valgrind,
                     **case}))
 
-            # combine pre-defines and per-case defines
-            self.defines = TEST_PREDEFINES + sorted(
+            # combine per-case defines
+            self.defines = sorted(
                 set.union(*(set(case.defines) for case in self.cases)))
 
             # combine other per-case things
@@ -225,216 +216,222 @@ def compile(**args):
                 % args['test_paths'])
             sys.exit(-1)
 
-        # write out a test suite
+        # load our suite
         suite = TestSuite(paths[0])
-        if 'output' in args:
-            with openio(args['output'], 'w') as f:
-                # redirect littlefs tracing
-                f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
-                f.write(8*' '+'extern FILE *test_trace; \\\n')
-                f.write(8*' '+'if (test_trace) { \\\n')
-                f.write(12*' '+'fprintf(test_trace, '
-                    '"%s:%d:trace: " fmt "%s\\n", \\\n')
-                f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
-                f.write(8*' '+'} \\\n')
-                f.write(4*' '+'} while (0)\n')
-                f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
-                f.write('#define LFS_TESTBD_TRACE(...) '
-                    'LFS_TRACE_(__VA_ARGS__, "")\n')
-                f.write('\n')
-
-                f.write('%s\n' % SUITE_PROLOGUE.strip())
-                f.write('\n')
+    else:
+        # load all suites
+        suites = [TestSuite(path) for path in paths]
+        suites.sort(key=lambda s: s.name)
+
+    # write generated test source
+    if 'output' in args:
+        with openio(args['output'], 'w') as f:
+            _write = f.write
+            def write(s):
+                f.lineno += s.count('\n')
+                _write(s)
+            def writeln(s=''):
+                f.lineno += s.count('\n') + 1
+                _write(s)
+                _write('\n')
+            f.lineno = 1
+            f.write = write
+            f.writeln = writeln
+
+            # redirect littlefs tracing
+            f.writeln('#define LFS_TRACE_(fmt, ...) do { \\')
+            f.writeln(8*' '+'extern FILE *test_trace; \\')
+            f.writeln(8*' '+'if (test_trace) { \\')
+            f.writeln(12*' '+'fprintf(test_trace, '
+                '"%s:%d:trace: " fmt "%s\\n", \\')
+            f.writeln(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\')
+            f.writeln(8*' '+'} \\')
+            f.writeln(4*' '+'} while (0)')
+            f.writeln('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")')
+            f.writeln('#define LFS_TESTBD_TRACE(...) '
+                'LFS_TRACE_(__VA_ARGS__, "")')
+            f.writeln()
+
+            if not args.get('source'):
+                # write test suite prologue
+                f.writeln('%s' % SUITE_PROLOGUE.strip())
+                f.writeln()
                 if suite.code is not None:
                     if suite.code_lineno is not None:
-                        f.write('#line %d "%s"\n'
+                        f.writeln('#line %d "%s"'
                             % (suite.code_lineno, suite.path))
                     f.write(suite.code)
-                    f.write('\n')
+                    if suite.code_lineno is not None:
+                        f.writeln('#line %d "%s"'
+                            % (f.lineno+1, args['output']))
+                    f.writeln()
 
-                for i, define in it.islice(
-                        enumerate(suite.defines),
-                        len(TEST_PREDEFINES), None):
-                    f.write('#define %-24s test_define(%d)\n' % (define, i))
-                f.write('\n')
+                for i, define in enumerate(suite.defines):
+                    f.writeln('#ifndef %s' % define)
+                    f.writeln('#define %-24s test_define(%d)' % (define, i))
+                    f.writeln('#endif')
+                f.writeln()
 
                 for case in suite.cases:
                     # create case defines
                     if case.defines:
                         sorted_defines = sorted(case.defines.items())
 
-                        for perm, defines in enumerate(
-                                it.product(*(
-                                    [(k, v) for v in vs]
-                                    for k, vs in sorted_defines))):
-                            f.write('const test_define_t '
-                                '__test__%s__%s__%d__defines[] = {\n'
-                                % (suite.name, case.name, perm))
-                            for k, v in defines:
-                                f.write(4*' '+'%s,\n' % v)
-                            f.write('};\n')
-                            f.write('\n')
-
-                        f.write('const test_define_t *const '
-                            '__test__%s__%s__defines[] = {\n'
+                        f.writeln('const test_define_t *const '
+                            '__test__%s__%s__defines[] = {'
                             % (suite.name, case.name))
-                        for perm in range(case.permutations):
-                            f.write(4*' '+'__test__%s__%s__%d__defines,\n'
-                                % (suite.name, case.name, perm))
-                        f.write('};\n')
-                        f.write('\n')
-
-                        f.write('const uint8_t '
-                            '__test__%s__%s__define_map[] = {\n'
+                        for defines in it.product(*(
+                                [(k, v) for v in vs]
+                                for k, vs in sorted_defines)):
+                            f.writeln(4*' '+'(const test_define_t[]){%s},'
+                                % ', '.join('%s' % v for _, v in defines)) 
+                        f.writeln('};')
+                        f.writeln()
+
+                        f.writeln('const uint8_t '
+                            '__test__%s__%s__define_map[] = {'
                             % (suite.name, case.name))
-                        for k in suite.defines:
-                            f.write(4*' '+'%s,\n'
-                                % ([k for k, _ in sorted_defines].index(k)
-                                    if k in case.defines else '0xff'))
-                        f.write('};\n')
-                        f.write('\n')
+                        f.writeln(4*' '+'%s,'
+                            % ', '.join(
+                                '%s' % [k for k, _ in sorted_defines].index(k)
+                                if k in case.defines else '0xff'
+                                for k in suite.defines))
+                        f.writeln('};')
+                        f.writeln()
 
                     # create case filter function
                     if suite.if_ is not None or case.if_ is not None:
-                        f.write('bool __test__%s__%s__filter('
-                            '__attribute__((unused)) uint32_t perm) {\n'
+                        f.writeln('bool __test__%s__%s__filter('
+                            '__attribute__((unused)) uint32_t perm) {'
                             % (suite.name, case.name))
                         if suite.if_ is not None:
-                            f.write(4*' '+'#line %d "%s"\n'
-                                % (suite.if_lineno, suite.path))
-                            f.write(4*' '+'if (!(%s)) {\n' % suite.if_)
-                            f.write(8*' '+'return false;\n')
-                            f.write(4*' '+'}\n')
-                            f.write('\n')
+                            if suite.if_lineno is not None:
+                                f.writeln(4*' '+'#line %d "%s"'
+                                    % (suite.if_lineno, suite.path))
+                            f.writeln(4*' '+'if (!(%s)) {' % suite.if_)
+                            if suite.if_lineno is not None:
+                                f.writeln(4*' '+'#line %d "%s"'
+                                    % (f.lineno+1, args['output']))
+                            f.writeln(8*' '+'return false;')
+                            f.writeln(4*' '+'}')
+                            f.writeln()
                         if case.if_ is not None:
-                            f.write(4*' '+'#line %d "%s"\n'
-                                % (case.if_lineno, suite.path))
-                            f.write(4*' '+'if (!(%s)) {\n' % case.if_)
-                            f.write(8*' '+'return false;\n')
-                            f.write(4*' '+'}\n')
-                            f.write('\n')
-                        f.write(4*' '+'return true;\n')
-                        f.write('}\n')
-                        f.write('\n')
+                            if case.if_lineno is not None:
+                                f.writeln(4*' '+'#line %d "%s"'
+                                    % (case.if_lineno, suite.path))
+                            f.writeln(4*' '+'if (!(%s)) {' % case.if_)
+                            if case.if_lineno is not None:
+                                f.writeln(4*' '+'#line %d "%s"'
+                                    % (f.lineno+1, args['output']))
+                            f.writeln(8*' '+'return false;')
+                            f.writeln(4*' '+'}')
+                            f.writeln()
+                        f.writeln(4*' '+'return true;')
+                        f.writeln('}')
+                        f.writeln()
 
                     # create case run function
-                    f.write('void __test__%s__%s__run('
+                    f.writeln('void __test__%s__%s__run('
                         '__attribute__((unused)) struct lfs_config *cfg, '
-                        '__attribute__((unused)) uint32_t perm) {\n'
+                        '__attribute__((unused)) uint32_t perm) {'
                         % (suite.name, case.name))
-                    f.write(4*' '+'%s\n'
-                        % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
-                    f.write('\n')
-                    f.write(4*' '+'// test case %s\n' % case.id())
+                    if CASE_PROLOGUE.strip():
+                        f.writeln(4*' '+'%s'
+                            % CASE_PROLOGUE.strip().replace('\n', '\n'+4*' '))
+                        f.writeln()
+                    f.writeln(4*' '+'// test case %s' % case.id())
                     if case.code_lineno is not None:
-                        f.write(4*' '+'#line %d "%s"\n'
+                        f.writeln(4*' '+'#line %d "%s"'
                             % (case.code_lineno, suite.path))
                     f.write(case.code)
-                    f.write('\n')
-                    f.write(4*' '+'%s\n'
-                        % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
-                    f.write('}\n')
-                    f.write('\n')
+                    if case.code_lineno is not None:
+                        f.writeln(4*' '+'#line %d "%s"'
+                            % (f.lineno+1, args['output']))
+                    if CASE_EPILOGUE.strip():
+                        f.writeln()
+                        f.writeln(4*' '+'%s'
+                            % CASE_EPILOGUE.strip().replace('\n', '\n'+4*' '))
+                    f.writeln('}')
+                    f.writeln()
 
                     # create case struct
-                    f.write('const struct test_case __test__%s__%s__case = {\n'
+                    f.writeln('const struct test_case __test__%s__%s__case = {'
                         % (suite.name, case.name))
-                    f.write(4*' '+'.id = "%s",\n' % case.id())
-                    f.write(4*' '+'.name = "%s",\n' % case.name)
-                    f.write(4*' '+'.path = "%s",\n' % case.path)
-                    f.write(4*' '+'.types = %s,\n'
+                    f.writeln(4*' '+'.id = "%s",' % case.id())
+                    f.writeln(4*' '+'.name = "%s",' % case.name)
+                    f.writeln(4*' '+'.path = "%s",' % case.path)
+                    f.writeln(4*' '+'.types = %s,'
                         % ' | '.join(filter(None, [
                             'TEST_NORMAL' if case.normal else None,
                             'TEST_REENTRANT' if case.reentrant else None,
                             'TEST_VALGRIND' if case.valgrind else None])))
-                    f.write(4*' '+'.permutations = %d,\n' % case.permutations)
+                    f.writeln(4*' '+'.permutations = %d,' % case.permutations)
                     if case.defines:
-                        f.write(4*' '+'.defines = __test__%s__%s__defines,\n'
+                        f.writeln(4*' '+'.defines = __test__%s__%s__defines,'
                             % (suite.name, case.name))
-                        f.write(4*' '+'.define_map = '
-                            '__test__%s__%s__define_map,\n'
+                        f.writeln(4*' '+'.define_map = '
+                            '__test__%s__%s__define_map,'
                             % (suite.name, case.name))
                     if suite.if_ is not None or case.if_ is not None:
-                        f.write(4*' '+'.filter = __test__%s__%s__filter,\n'
+                        f.writeln(4*' '+'.filter = __test__%s__%s__filter,'
                             % (suite.name, case.name))
-                    f.write(4*' '+'.run = __test__%s__%s__run,\n'
+                    f.writeln(4*' '+'.run = __test__%s__%s__run,'
                         % (suite.name, case.name))
-                    f.write('};\n')
-                    f.write('\n')
+                    f.writeln('};')
+                    f.writeln()
 
                 # create suite define names
-                f.write('const char *const __test__%s__define_names[] = {\n'
+                f.writeln('const char *const __test__%s__define_names[] = {'
                     % suite.name)
                 for k in suite.defines:
-                    f.write(4*' '+'"%s",\n' % k)
-                f.write('};\n')
-                f.write('\n')
+                    f.writeln(4*' '+'"%s",' % k)
+                f.writeln('};')
+                f.writeln()
 
                 # create suite struct
-                f.write('const struct test_suite __test__%s__suite = {\n'
+                f.writeln('const struct test_suite __test__%s__suite = {'
                     % suite.name)
-                f.write(4*' '+'.id = "%s",\n' % suite.id())
-                f.write(4*' '+'.name = "%s",\n' % suite.name)
-                f.write(4*' '+'.path = "%s",\n' % suite.path)
-                f.write(4*' '+'.types = %s,\n'
+                f.writeln(4*' '+'.id = "%s",' % suite.id())
+                f.writeln(4*' '+'.name = "%s",' % suite.name)
+                f.writeln(4*' '+'.path = "%s",' % suite.path)
+                f.writeln(4*' '+'.types = %s,'
                     % ' | '.join(filter(None, [
                         'TEST_NORMAL' if suite.normal else None,
                         'TEST_REENTRANT' if suite.reentrant else None,
                         'TEST_VALGRIND' if suite.valgrind else None])))
-                f.write(4*' '+'.define_names = __test__%s__define_names,\n'
+                f.writeln(4*' '+'.define_names = __test__%s__define_names,'
                     % suite.name)
-                f.write(4*' '+'.define_count = %d,\n' % len(suite.defines))
-                f.write(4*' '+'.cases = (const struct test_case *const []){\n')
+                f.writeln(4*' '+'.define_count = %d,' % len(suite.defines))
+                f.writeln(4*' '+'.cases = (const struct test_case *const []){')
                 for case in suite.cases:
-                    f.write(8*' '+'&__test__%s__%s__case,\n'
+                    f.writeln(8*' '+'&__test__%s__%s__case,'
                         % (suite.name, case.name))
-                f.write(4*' '+'},\n')
-                f.write(4*' '+'.case_count = %d,\n' % len(suite.cases))
-                f.write('};\n')
-                f.write('\n')
-
-    else:
-        # load all suites
-        suites = [TestSuite(path) for path in paths]
-        suites.sort(key=lambda s: s.name)
-
-        # write out a test source
-        if 'output' in args:
-            with openio(args['output'], 'w') as f:
-                # redirect littlefs tracing
-                f.write('#define LFS_TRACE_(fmt, ...) do { \\\n')
-                f.write(8*' '+'extern FILE *test_trace; \\\n')
-                f.write(8*' '+'if (test_trace) { \\\n')
-                f.write(12*' '+'fprintf(test_trace, '
-                    '"%s:%d:trace: " fmt "%s\\n", \\\n')
-                f.write(20*' '+'__FILE__, __LINE__, __VA_ARGS__); \\\n')
-                f.write(8*' '+'} \\\n')
-                f.write(4*' '+'} while (0)\n')
-                f.write('#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")\n')
-                f.write('#define LFS_TESTBD_TRACE(...) '
-                    'LFS_TRACE_(__VA_ARGS__, "")\n')
-                f.write('\n')
+                f.writeln(4*' '+'},')
+                f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
+                f.writeln('};')
+                f.writeln()
 
+            else:
                 # copy source
-                f.write('#line 1 "%s"\n' % args['source'])
+                f.writeln('#line 1 "%s"' % args['source'])
                 with open(args['source']) as sf:
                     shutil.copyfileobj(sf, f)
-                f.write('\n')
+                f.writeln()
 
                 f.write(SUITE_PROLOGUE)
-                f.write('\n')
+                f.writeln()
 
                 # add suite info to test_runner.c
                 if args['source'] == 'runners/test_runner.c':
-                    f.write('\n')
+                    f.writeln()
                     for suite in suites:
-                        f.write('extern const struct test_suite '
-                            '__test__%s__suite;\n' % suite.name)
-                    f.write('const struct test_suite *test_suites[] = {\n')
+                        f.writeln('extern const struct test_suite '
+                            '__test__%s__suite;' % suite.name)
+                    f.writeln('const struct test_suite *test_suites[] = {')
                     for suite in suites:
-                        f.write(4*' '+'&__test__%s__suite,\n' % suite.name)
-                    f.write('};\n')
-                    f.write('const size_t test_suite_count = %d;\n'
+                        f.writeln(4*' '+'&__test__%s__suite,' % suite.name)
+                    f.writeln('};')
+                    f.writeln('const size_t test_suite_count = %d;'
                         % len(suites))
 
 def runner(**args):
@@ -469,7 +466,7 @@ def list_(**args):
 
 
 def find_cases(runner_, **args):
-    # first get suite/case/perm counts
+    # query from runner
     cmd = runner_ + ['--list-cases']
     if args.get('verbose'):
         print(' '.join(shlex.quote(c) for c in cmd))
@@ -483,9 +480,8 @@ def find_cases(runner_, **args):
     expected_perms = 0
     total_perms = 0
     pattern = re.compile(
-        '^(?P<id>(?P<suite>[^#]+)#[^ #]+) +'
-            '[^ ]+ +[^ ]+ +[^ ]+ +'
-            '(?P<filtered>[0-9]+)/(?P<perms>[0-9]+)$')
+        '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
+            '[^\s]+\s+(?P<filtered>\d+)/(?P<perms>\d+)')
     # skip the first line
     next(proc.stdout)
     for line in proc.stdout:
@@ -509,11 +505,69 @@ def find_cases(runner_, **args):
         expected_perms,
         total_perms)
 
+def find_paths(runner_, **args):
+    # query from runner
+    cmd = runner_ + ['--list-paths']
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace')
+    paths = co.OrderedDict()
+    pattern = re.compile(
+        '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
+            '(?P<path>[^:]+):(?P<lineno>\d+)')
+    # skip the first line
+    for line in proc.stdout:
+        m = pattern.match(line)
+        if m:
+            paths[m.group('id')] = (m.group('path'), int(m.group('lineno')))
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    return paths
+
+def find_defines(runner_, **args):
+    # query from runner
+    cmd = runner_ + ['--list-defines']
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace')
+    defines = co.OrderedDict()
+    pattern = re.compile(
+        '^(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)\s+'
+            '(?P<defines>(?:\w+=\w+\s*)+)')
+    # skip the first line
+    for line in proc.stdout:
+        m = pattern.match(line)
+        if m:
+            defines[m.group('id')] = {k: v
+                for k, v in re.findall('(\w+)=(\w+)', m.group('defines'))}
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    return defines
+
+
 class TestFailure(Exception):
-    def __init__(self, id, returncode, stdout, assert_=None):
+    def __init__(self, id, returncode, output, assert_=None):
         self.id = id
         self.returncode = returncode
-        self.stdout = stdout
+        self.output = output
         self.assert_ = assert_
 
 def run_step(name, runner_, **args):
@@ -531,7 +585,7 @@ def run_step(name, runner_, **args):
     pattern = re.compile('^(?:'
             '(?P<op>running|finished|skipped) '
                 '(?P<id>(?P<case>(?P<suite>[^#]+)#[^\s#]+)[^\s]*)'
-            '|' '(?P<path>[^:]+):(?P<lineno>[0-9]+):(?P<op_>assert):'
+            '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
                 ' *(?P<message>.*)' ')$')
     locals = th.local()
     # TODO use process group instead of this set?
@@ -554,7 +608,7 @@ def run_step(name, runner_, **args):
         children.add(proc)
 
         last_id = None
-        last_stdout = []
+        last_output = []
         last_assert = None
         try:
             while True:
@@ -567,7 +621,7 @@ def run_step(name, runner_, **args):
                     raise
                 if not line:
                     break
-                last_stdout.append(line)
+                last_output.append(line)
                 if args.get('verbose'):
                     sys.stdout.write(line)
 
@@ -577,7 +631,7 @@ def run_step(name, runner_, **args):
                     if op == 'running':
                         locals.seen_perms += 1
                         last_id = m.group('id')
-                        last_stdout = []
+                        last_output = []
                         last_assert = None
                     elif op == 'finished':
                         passed_suite_perms[m.group('suite')] += 1
@@ -590,10 +644,11 @@ def run_step(name, runner_, **args):
                             m.group('path'),
                             int(m.group('lineno')),
                             m.group('message'))
-                        # TODO why is kill _so_ much faster than terminate?
-                        proc.kill()
+                        # go ahead and kill the process, aborting takes a while
+                        if args.get('keep_going'):
+                            proc.kill()
         except KeyboardInterrupt:
-            raise TestFailure(last_id, 1, last_stdout)
+            raise TestFailure(last_id, 1, last_output)
         finally:
             children.remove(proc)
             mpty.close()
@@ -603,7 +658,7 @@ def run_step(name, runner_, **args):
             raise TestFailure(
                 last_id,
                 proc.returncode,
-                last_stdout,
+                last_output,
                 last_assert)
 
     def run_job(runner, skip=None, every=None):
@@ -636,7 +691,6 @@ def run_step(name, runner_, **args):
                 else:
                     # stop other tests
                     for child in children:
-                        # TODO why is kill _so_ much faster than terminate?
                         child.kill()
 
             break
@@ -759,11 +813,28 @@ def run(**args):
     print()
 
     # print each failure
-    # TODO get line, defines, path
+    if failures:
+        # get some extra info from runner
+        runner_paths = find_paths(runner_, **args)
+        runner_defines = find_defines(runner_, **args)
+
     for failure in failures:
-#        print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s failed'
-#            # TODO this should be the suite path and lineno
-#            % (failure.assert
+        # show summary of failure
+        path, lineno = runner_paths[testcase(failure.id)]
+        defines = runner_defines[failure.id]
+
+        print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed'
+            % (path, lineno, failure.id,
+                ' (%s)' % ', '.join(
+                    '%s=%s' % (k, v) for k, v in defines.items())
+                if defines else ''))
+
+        if failure.output:
+            output = failure.output
+            if failure.assert_ is not None:
+                output = output[:-1]
+            for line in output[-5:]:
+                sys.stdout.write(line)
 
         if failure.assert_ is not None:
             path, lineno, message = failure.assert_
@@ -785,7 +856,8 @@ def main(**args):
             or args.get('list_cases')
             or args.get('list_paths')
             or args.get('list_defines')
-            or args.get('list_geometries')):
+            or args.get('list_geometries')
+            or args.get('list_defaults')):
         list_(**args)
     else:
         run(**args)
@@ -816,6 +888,8 @@ if __name__ == "__main__":
         help="List the defines for each test permutation.")
     test_parser.add_argument('--list-geometries', action='store_true',
         help="List the disk geometries used for testing.")
+    test_parser.add_argument('--list-defaults', action='store_true',
+        help="List the default defines in this test-runner.")
     test_parser.add_argument('-D', '--define', action='append',
         help="Override a test define.")
     test_parser.add_argument('-G', '--geometry',