Dropped namespacing of test cases

The main benefit is small test ids everywhere, though this is with the
downside of needing longer names to properly prefix and avoid
collisions. But this fits into the rest of the scripts with globally
unique names a bit better. This is a C project after all.

The other small benefit is test generators may have an easier time since
per-case symbols can expect to be unique.

runners/test_runner.c

@@ -110,8 +110,7 @@ typedef struct test_powerloss {
 } test_powerloss_t;
 
 typedef struct test_id {
-    const char *suite;
-    const char *case_;
+    const char *name;
     const test_define_t *defines;
     size_t define_count;
     const lfs_testbd_powercycles_t *cycles;
@@ -415,7 +414,7 @@ extern const test_powerloss_t *test_powerlosses;
 extern size_t test_powerloss_count;
 
 const test_id_t *test_ids = (const test_id_t[]) {
-    {NULL, NULL, NULL, 0, NULL, 0},
+    {NULL, NULL, 0, NULL, 0},
 };
 size_t test_id_count = 1;
 
@@ -489,8 +488,8 @@ static void perm_printid(
         const lfs_testbd_powercycles_t *cycles,
         size_t cycle_count) {
     (void)suite;
-    // suite[:case[:permutation[:powercycles]]]]
-    printf("%s:", case_->id);
+    // case[:permutation[:powercycles]]]
+    printf("%s:", case_->name);
     for (size_t d = 0;
             d < lfs_max(
                 suite->define_count,
@@ -625,16 +624,15 @@ static void summary(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -674,19 +672,18 @@ static void list_suites(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             size_t cases = 0;
             struct perm_count_state perms = {0, 0};
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -702,6 +699,11 @@ static void list_suites(void) {
                         &perms);
             }
 
+            // no tests found?
+            if (!cases) {
+                continue;
+            }
+
             char perm_buf[64];
             sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
             char flag_buf[64];
@@ -709,7 +711,7 @@ static void list_suites(void) {
                     (test_suites[i].flags & TEST_REENTRANT) ? "r" : "",
                     (!test_suites[i].flags) ? "-" : "");
             printf("%-36s %7s %7zu %11s\n",
-                    test_suites[i].id,
+                    test_suites[i].name,
                     flag_buf,
                     cases,
                     perm_buf);
@@ -722,16 +724,15 @@ static void list_cases(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -755,7 +756,7 @@ static void list_cases(void) {
                         (!test_suites[i].cases[j].flags)
                             ? "-" : "");
                 printf("%-36s %7s %11s\n",
-                        test_suites[i].cases[j].id,
+                        test_suites[i].cases[j].name,
                         flag_buf,
                         perm_buf);
             }
@@ -768,13 +769,26 @@ static void list_suite_paths(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
+            size_t cases = 0;
+
+            for (size_t j = 0; j < test_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+            }
+
+            // no tests found?
+            if (!cases) {
                 continue;
             }
 
             printf("%-36s %s\n",
-                    test_suites[i].id,
+                    test_suites[i].name,
                     test_suites[i].path);
         }
     }
@@ -785,19 +799,18 @@ static void list_case_paths(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
                 printf("%-36s %s\n",
-                        test_suites[i].cases[j].id,
+                        test_suites[i].cases[j].name,
                         test_suites[i].cases[j].path);
             }
         }
@@ -907,16 +920,15 @@ static void list_defines(void) {
     // add defines
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -956,16 +968,15 @@ static void list_permutation_defines(void) {
     // add permutation defines
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -1637,16 +1648,15 @@ static void run(void) {
 
     for (size_t t = 0; t < test_id_count; t++) {
         for (size_t i = 0; i < TEST_SUITE_COUNT; i++) {
-            if (test_ids[t].suite && strcmp(
-                    test_suites[i].name, test_ids[t].suite) != 0) {
-                continue;
-            }
-
             test_define_suite(&test_suites[i]);
 
             for (size_t j = 0; j < test_suites[i].case_count; j++) {
-                if (test_ids[t].case_ && strcmp(
-                        test_suites[i].cases[j].name, test_ids[t].case_) != 0) {
+                // does neither suite nor case name match?
+                if (test_ids[t].name && !(
+                        strcmp(test_ids[t].name,
+                            test_suites[i].name) == 0
+                        || strcmp(test_ids[t].name,
+                            test_suites[i].cases[j].name) == 0)) {
                     continue;
                 }
 
@@ -2367,83 +2377,72 @@ getopt_done: ;
         lfs_testbd_powercycles_t *cycles = NULL;
         size_t cycle_count = 0;
 
-        // parse suite
-        char *suite = argv[optind];
-        char *case_ = strchr(suite, ':');
-        if (case_) {
-            *case_ = '\0';
-            case_ += 1;
+        // parse name, can be suite or case
+        char *name = argv[optind];
+        char *defines_ = strchr(name, ':');
+        if (defines_) {
+            *defines_ = '\0';
+            defines_ += 1;
         }
 
         // remove optional path and .toml suffix
-        char *slash = strrchr(suite, '/');
+        char *slash = strrchr(name, '/');
         if (slash) {
-            suite = slash+1;
+            name = slash+1;
         }
 
-        size_t suite_len = strlen(suite);
-        if (suite_len > 5 && strcmp(&suite[suite_len-5], ".toml") == 0) {
-            suite[suite_len-5] = '\0';
+        size_t name_len = strlen(name);
+        if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) {
+            name[name_len-5] = '\0';
         }
 
-        if (case_) {
-            // parse case
-            char *defines_ = strchr(case_, ':');
-            if (defines_) {
-                *defines_ = '\0';
-                defines_ += 1;
+        if (defines_) {
+            // parse defines
+            char *cycles_ = strchr(defines_, ':');
+            if (cycles_) {
+                *cycles_ = '\0';
+                cycles_ += 1;
             }
 
-            // nothing really to do for case
-
-            if (defines_) {
-                // parse defines
-                char *cycles_ = strchr(defines_, ':');
-                if (cycles_) {
-                    *cycles_ = '\0';
-                    cycles_ += 1;
+            while (true) {
+                char *parsed;
+                size_t d = leb16_parse(defines_, &parsed);
+                intmax_t v = leb16_parse(parsed, &parsed);
+                if (parsed == defines_) {
+                    break;
                 }
-
-                while (true) {
-                    char *parsed;
-                    size_t d = leb16_parse(defines_, &parsed);
-                    intmax_t v = leb16_parse(parsed, &parsed);
-                    if (parsed == defines_) {
-                        break;
-                    }
-                    defines_ = parsed;
-
-                    if (d >= define_count) {
-                        // align to power of two to avoid any superlinear growth
-                        size_t ncount = 1 << lfs_npw2(d+1);
-                        defines = realloc(defines,
-                                ncount*sizeof(test_define_t));
-                        memset(defines+define_count, 0,
-                                (ncount-define_count)*sizeof(test_define_t));
-                        define_count = ncount;
-                    }
-                    defines[d] = (test_define_t)TEST_LIT(v);
+                defines_ = parsed;
+
+                if (d >= define_count) {
+                    // align to power of two to avoid any superlinear growth
+                    size_t ncount = 1 << lfs_npw2(d+1);
+                    defines = realloc(defines,
+                            ncount*sizeof(test_define_t));
+                    memset(defines+define_count, 0,
+                            (ncount-define_count)*sizeof(test_define_t));
+                    define_count = ncount;
                 }
+                defines[d] = (test_define_t)TEST_LIT(v);
+            }
 
-                if (cycles_) {
-                    // parse power cycles
-                    size_t cycle_capacity = 0;
-                    while (*cycles_ != '\0') {
-                        char *parsed = NULL;
-                        *(lfs_testbd_powercycles_t*)mappend(
-                                (void**)&cycles,
-                                sizeof(lfs_testbd_powercycles_t),
-                                &cycle_count,
-                                &cycle_capacity)
-                                = leb16_parse(cycles_, &parsed);
-                        if (parsed == cycles_) {
-                            fprintf(stderr, "error: "
-                                    "could not parse test cycles: %s\n",
-                                    cycles_);
-                            exit(-1);
-                        }
-                        cycles_ = parsed;
+            if (cycles_) {
+                // parse power cycles
+                size_t cycle_capacity = 0;
+                while (*cycles_ != '\0') {
+                    char *parsed = NULL;
+                    *(lfs_testbd_powercycles_t*)mappend(
+                            (void**)&cycles,
+                            sizeof(lfs_testbd_powercycles_t),
+                            &cycle_count,
+                            &cycle_capacity)
+                            = leb16_parse(cycles_, &parsed);
+                    if (parsed == cycles_) {
+                        fprintf(stderr, "error: "
+                                "could not parse test cycles: %s\n",
+                                cycles_);
+                        exit(-1);
                     }
+                    cycles_ = parsed;
                 }
             }
         }
@@ -2454,8 +2453,7 @@ getopt_done: ;
                 sizeof(test_id_t),
                 &test_id_count,
                 &test_id_capacity) = (test_id_t){
-            .suite = suite,
-            .case_ = case_,
+            .name = name,
             .defines = defines,
             .define_count = define_count,
             .cycles = cycles,

runners/test_runner.h

@@ -37,7 +37,6 @@ typedef struct test_define {
 } test_define_t;
 
 struct test_case {
-    const char *id;
     const char *name;
     const char *path;
     test_flags_t flags;
@@ -50,7 +49,6 @@ struct test_case {
 };
 
 struct test_suite {
-    const char *id;
     const char *name;
     const char *path;
     test_flags_t flags;

scripts/test.py

@@ -133,13 +133,10 @@ class TestCase:
             print('%swarning:%s in %s, found unused key %r' % (
                 '\x1b[01;33m' if args['color'] else '',
                 '\x1b[m' if args['color'] else '',
-                self.id(),
+                self.name,
                 k),
                 file=sys.stderr)
 
-    def id(self):
-        return '%s:%s' % (self.suite, self.name)
-
 
 class TestSuite:
     # create a TestSuite object from a toml file
@@ -221,13 +218,10 @@ class TestSuite:
             print('%swarning:%s in %s, found unused key %r' % (
                 '\x1b[01;33m' if args['color'] else '',
                 '\x1b[m' if args['color'] else '',
-                self.id(),
+                self.name,
                 k),
                 file=sys.stderr)
 
-    def id(self):
-        return self.name
-
 
 
 def compile(test_paths, **args):
@@ -244,17 +238,45 @@ def compile(test_paths, **args):
         print('no test suites found in %r?' % test_paths)
         sys.exit(-1)
 
+    # load the suites
+    suites = [TestSuite(path, args) for path in paths]
+    suites.sort(key=lambda s: s.name)
+
+    # check for name conflicts, these will cause ambiguity problems later
+    # when running tests
+    seen = {}
+    for suite in suites:
+        if suite.name in seen:
+            print('%swarning:%s conflicting suite %r, %s and %s' % (
+                '\x1b[01;33m' if args['color'] else '',
+                '\x1b[m' if args['color'] else '',
+                suite.name,
+                suite.path,
+                seen[suite.name].path),
+                file=sys.stderr)
+        seen[suite.name] = suite
+
+        for case in suite.cases:
+            # only allow conflicts if a case and its suite share a name
+            if case.name in seen and not (
+                    isinstance(seen[case.name], TestSuite)
+                    and seen[case.name].cases == [case]):
+                print('%swarning:%s conflicting case %r, %s and %s' % (
+                    '\x1b[01;33m' if args['color'] else '',
+                    '\x1b[m' if args['color'] else '',
+                    case.name,
+                    case.path,
+                    seen[case.name].path),
+                    file=sys.stderr)
+            seen[case.name] = case
+
+    # we can only compile one test suite at a time
     if not args.get('source'):
-        if len(paths) > 1:
+        if len(suites) > 1:
             print('more than one test suite for compilation? (%r)' % test_paths)
             sys.exit(-1)
 
-        # load our suite
-        suite = TestSuite(paths[0], args)
-    else:
-        # load all suites
-        suites = [TestSuite(path, args) for path in paths]
-        suites.sort(key=lambda s: s.name)
+        suite = suites[0]
 
     # write generated test source
     if 'output' in args:
@@ -332,7 +354,7 @@ def compile(test_paths, **args):
                     f.writeln('void __test__%s__%s__run('
                         '__attribute__((unused)) struct lfs_config *cfg) {'
                         % (suite.name, case.name))
-                    f.writeln(4*' '+'// test case %s' % case.id())
+                    f.writeln(4*' '+'// test case %s' % case.name)
                     if case.code_lineno is not None:
                         f.writeln(4*' '+'#line %d "%s"'
                             % (case.code_lineno, suite.path))
@@ -384,10 +406,14 @@ def compile(test_paths, **args):
                         f.writeln()
 
                 # create suite struct
-                f.writeln('__attribute__((section("_test_suites")))')
+                #
+                # note we place this in the custom test_suites section with
+                # minimum alignment, otherwise GCC ups the alignment to
+                # 32-bytes for some reason
+                f.writeln('__attribute__((section("_test_suites"), '
+                    'aligned(1)))')
                 f.writeln('const struct test_suite __test__%s__suite = {'
                     % suite.name)
-                f.writeln(4*' '+'.id = "%s",' % suite.id())
                 f.writeln(4*' '+'.name = "%s",' % suite.name)
                 f.writeln(4*' '+'.path = "%s",' % suite.path)
                 f.writeln(4*' '+'.flags = %s,'
@@ -408,7 +434,6 @@ def compile(test_paths, **args):
                 for case in suite.cases:
                     # create case structs
                     f.writeln(8*' '+'{')
-                    f.writeln(12*' '+'.id = "%s",' % case.id())
                     f.writeln(12*' '+'.name = "%s",' % case.name)
                     f.writeln(12*' '+'.path = "%s",' % case.path)
                     f.writeln(12*' '+'.flags = %s,'
@@ -534,8 +559,13 @@ def list_(runner, test_ids=[], **args):
     return sp.call(cmd)
 
 
-def find_cases(runner_, ids=[], **args):
-    # query from runner
+def find_perms(runner_, ids=[], **args):
+    case_suites = {}
+    expected_case_perms = co.defaultdict(lambda: 0)
+    expected_perms = 0
+    total_perms = 0
+
+    # query cases from the runner
     cmd = runner_ + ['--list-cases'] + ids
     if args.get('verbose'):
         print(' '.join(shlex.quote(c) for c in cmd))
@@ -545,21 +575,17 @@ def find_cases(runner_, ids=[], **args):
         universal_newlines=True,
         errors='replace',
         close_fds=False)
-    expected_suite_perms = co.defaultdict(lambda: 0)
-    expected_case_perms = co.defaultdict(lambda: 0)
-    expected_perms = 0
-    total_perms = 0
     pattern = re.compile(
-        '^(?P<id>(?P<case>(?P<suite>[^:]+):[^\s:]+)[^\s]*)\s+'
-            '[^\s]+\s+(?P<filtered>\d+)/(?P<perms>\d+)')
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<flags>[^\s]+)'
+            '\s+(?P<filtered>\d+)/(?P<perms>\d+)')
     # skip the first line
     for line in it.islice(proc.stdout, 1, None):
         m = pattern.match(line)
         if m:
             filtered = int(m.group('filtered'))
             perms = int(m.group('perms'))
-            expected_suite_perms[m.group('suite')] += filtered
-            expected_case_perms[m.group('id')] += filtered
+            expected_case_perms[m.group('case')] += filtered
             expected_perms += filtered
             total_perms += perms
     proc.wait()
@@ -569,13 +595,50 @@ def find_cases(runner_, ids=[], **args):
                 sys.stdout.write(line)
         sys.exit(-1)
 
+    # get which suite each case belongs to via paths
+    cmd = runner_ + ['--list-case-paths'] + ids
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    pattern = re.compile(
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
+    # skip the first line
+    for line in it.islice(proc.stdout, 1, None):
+        m = pattern.match(line)
+        if m:
+            path = m.group('path')
+            # strip path/suffix here
+            suite = os.path.basename(path)
+            if suite.endswith('.toml'):
+                suite = suite[:-len('.toml')]
+            case_suites[m.group('case')] = suite
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    # figure out expected suite perms
+    expected_suite_perms = co.defaultdict(lambda: 0)
+    for case, suite in case_suites.items():
+        expected_suite_perms[suite] += expected_case_perms[case]
+
     return (
+        case_suites,
         expected_suite_perms,
         expected_case_perms,
         expected_perms,
         total_perms)
 
 def find_path(runner_, id, **args):
+    path = None
     # query from runner
     cmd = runner_ + ['--list-case-paths', id]
     if args.get('verbose'):
@@ -586,10 +649,9 @@ def find_path(runner_, id, **args):
         universal_newlines=True,
         errors='replace',
         close_fds=False)
-    path = None
     pattern = re.compile(
-        '^(?P<id>(?P<case>(?P<suite>[^:]+):[^\s:]+)[^\s]*)\s+'
-            '(?P<path>[^:]+):(?P<lineno>\d+)')
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
     # skip the first line
     for line in it.islice(proc.stdout, 1, None):
         m = pattern.match(line)
@@ -680,8 +742,11 @@ class TestFailure(Exception):
 
 def run_stage(name, runner_, ids, output_, **args):
     # get expected suite/case/perm counts
-    expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
-        find_cases(runner_, ids, **args))
+    (case_suites,
+        expected_suite_perms,
+        expected_case_perms,
+        expected_perms,
+        total_perms) = find_perms(runner_, ids, **args)
 
     passed_suite_perms = co.defaultdict(lambda: 0)
     passed_case_perms = co.defaultdict(lambda: 0)
@@ -692,9 +757,10 @@ def run_stage(name, runner_, ids, output_, **args):
 
     pattern = re.compile('^(?:'
             '(?P<op>running|finished|skipped|powerloss) '
-                '(?P<id>(?P<case>(?P<suite>[^:]+):[^\s:]+)[^\s]*)'
+                '(?P<id>(?P<case>[^:]+)[^\s]*)'
             '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
-                ' *(?P<message>.*)' ')$')
+                ' *(?P<message>.*)'
+        ')$')
     locals = th.local()
     children = set()
 
@@ -759,15 +825,18 @@ def run_stage(name, runner_, ids, output_, **args):
                         last_id = m.group('id')
                         powerlosses += 1
                     elif op == 'finished':
-                        passed_suite_perms[m.group('suite')] += 1
-                        passed_case_perms[m.group('case')] += 1
+                        case = m.group('case')
+                        suite = case_suites[case]
+                        passed_suite_perms[suite] += 1
+                        passed_case_perms[case] += 1
                         passed_perms += 1
                         if output_:
                             # get defines and write to csv
                             defines = find_defines(
                                 runner_, m.group('id'), **args)
                             output_.writerow({
-                                'case': m.group('case'),
+                                'suite': suite,
+                                'case': case,
                                 'test_passed': '1/1',
                                 **defines})
                     elif op == 'skipped':
@@ -818,11 +887,13 @@ def run_stage(name, runner_, ids, output_, **args):
             except TestFailure as failure:
                 # keep track of failures
                 if output_:
-                    suite, case, _ = failure.id.split(':', 2)
+                    case, _ = failure.id.split(':', 1)
+                    suite = case_suites[case]
                     # get defines and write to csv
                     defines = find_defines(runner_, failure.id, **args)
                     output_.writerow({
-                        'case': ':'.join([suite, case]),
+                        'suite': suite,
+                        'case': case,
                         'test_passed': '0/1',
                         **defines})
 
@@ -919,13 +990,16 @@ def run(runner, test_ids=[], **args):
     # query runner for tests
     runner_ = find_runner(runner, **args)
     print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_))
-    expected_suite_perms, expected_case_perms, expected_perms, total_perms = (
-        find_cases(runner_, test_ids, **args))
-    print('found %d suites, %d cases, %d/%d permutations'
-        % (len(expected_suite_perms),
-            len(expected_case_perms),
-            expected_perms,
-            total_perms))
+    (_,
+        expected_suite_perms,
+        expected_case_perms,
+        expected_perms,
+        total_perms) = find_perms(runner_, test_ids, **args)
+    print('found %d suites, %d cases, %d/%d permutations' % (
+        len(expected_suite_perms),
+        len(expected_case_perms),
+        expected_perms,
+        total_perms))
     print()
 
     # truncate and open logs here so they aren't disconnected between tests
@@ -937,7 +1011,7 @@ def run(runner, test_ids=[], **args):
         trace = openio(args['trace'], 'w', 1)
     output = None
     if args.get('output'):
-        output = TestOutput(args['output'], ['case'], ['test_passed'])
+        output = TestOutput(args['output'], ['suite', 'case'], ['test_passed'])
 
     # measure runtime
     start = time.time()
@@ -951,12 +1025,17 @@ def run(runner, test_ids=[], **args):
             else expected_suite_perms.keys() if args.get('by_suites')
             else [None]):
         # spawn jobs for stage
-        expected_, passed_, powerlosses_, failures_, killed = run_stage(
-            by or 'tests',
-            runner_,
-            [by] if by is not None else test_ids,
-            output,
-            **args)
+        (expected_,
+            passed_,
+            powerlosses_,
+            failures_,
+            killed) = run_stage(
+                by or 'tests',
+                runner_,
+                [by] if by is not None else test_ids,
+                output,
+                **args)
+        # collect passes/failures
         expected += expected_
         passed += passed_
         powerlosses += powerlosses_

tests/test_alloc.toml

@@ -3,7 +3,7 @@
 if = 'BLOCK_CYCLES == -1'
 
 # parallel allocation test
-[cases.parallel_allocation]
+[cases.test_alloc_parallel]
 defines.FILES = 3
 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)'
 code = '''
@@ -52,7 +52,7 @@ code = '''
 '''
 
 # serial allocation test
-[cases.serial_allocation]
+[cases.test_alloc_serial]
 defines.FILES = 3
 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)'
 code = '''
@@ -99,7 +99,7 @@ code = '''
 '''
 
 # parallel allocation reuse test
-[cases.parallel_allocation_reuse]
+[cases.test_alloc_parallel_reuse]
 defines.FILES = 3
 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)'
 defines.CYCLES = [1, 10]
@@ -161,7 +161,7 @@ code = '''
 '''
 
 # serial allocation reuse test
-[cases.serial_allocation_reuse]
+[cases.test_alloc_serial_reuse]
 defines.FILES = 3
 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-6)) / FILES)'
 defines.CYCLES = [1, 10]
@@ -221,7 +221,7 @@ code = '''
 '''
 
 # exhaustion test
-[cases.exhaustion]
+[cases.test_alloc_exhaustion]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -261,7 +261,7 @@ code = '''
 '''
 
 # exhaustion wraparound test
-[cases.exhaustion_wraparound]
+[cases.test_alloc_exhaustion_wraparound]
 defines.SIZE = '(((BLOCK_SIZE-8)*(BLOCK_COUNT-4)) / 3)'
 code = '''
     lfs_t lfs;
@@ -313,7 +313,7 @@ code = '''
 '''
 
 # dir exhaustion test
-[cases.dir_exhaustion]
+[cases.test_alloc_dir_exhaustion]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -367,7 +367,7 @@ code = '''
 '''
 
 # what if we have a bad block during an allocation scan?
-[cases.bad_block_allocation]
+[cases.test_alloc_bad_blocks]
 in = "lfs.c"
 defines.ERASE_CYCLES = 0xffffffff
 defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR'
@@ -459,7 +459,7 @@ code = '''
 # should be removed and replaced with generalized tests.
 
 # chained dir exhaustion test
-[cases.chained_dir_exhaustion]
+[cases.test_alloc_chained_dir_exhaustion]
 if = 'BLOCK_SIZE == 512'
 defines.BLOCK_COUNT = 1024
 code = '''
@@ -537,7 +537,7 @@ code = '''
 '''
 
 # split dir test
-[cases.split_dir]
+[cases.test_alloc_split_dir]
 if = 'BLOCK_SIZE == 512'
 defines.BLOCK_COUNT = 1024
 code = '''
@@ -586,7 +586,7 @@ code = '''
 '''
 
 # outdated lookahead test
-[cases.outdated_lookahead]
+[cases.test_alloc_outdated_lookahead]
 if = 'BLOCK_SIZE == 512'
 defines.BLOCK_COUNT = 1024
 code = '''
@@ -654,7 +654,7 @@ code = '''
 '''
 
 # outdated lookahead and split dir test
-[cases.outdated_lookahead_split_dir]
+[cases.test_alloc_outdated_lookahead_split_dir]
 if = 'BLOCK_SIZE == 512'
 defines.BLOCK_COUNT = 1024
 code = '''

tests/test_attrs.toml

@@ -1,4 +1,4 @@
-[cases.get_set_attrs]
+[cases.test_attrs_get_set]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -79,7 +79,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.get_set_root_attrs]
+[cases.test_attrs_get_set_root]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -159,7 +159,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.get_set_file_attrs]
+[cases.test_attrs_get_set_file]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -269,7 +269,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.deferred_file_attrs]
+[cases.test_attrs_deferred_file]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;

tests/test_badblocks.toml

@@ -1,7 +1,7 @@
 # bad blocks with block cycles should be tested in test_relocations
 if = '(int32_t)BLOCK_CYCLES == -1'
 
-[cases.single_bad_blocks]
+[cases.test_badblocks_single]
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
@@ -81,7 +81,7 @@ code = '''
     }
 '''
 
-[cases.region_corruption] # (causes cascading failures)
+[cases.test_badblocks_region_corruption] # (causes cascading failures)
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
@@ -160,7 +160,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.alternating_corruption] # (causes cascading failures)
+[cases.test_badblocks_alternating_corruption] # (causes cascading failures)
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
@@ -240,7 +240,7 @@ code = '''
 '''
 
 # other corner cases
-[cases.bad_superblocks] # (corrupt 1 or 0)
+[cases.test_badblocks_superblocks] # (corrupt 1 or 0)
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
 defines.BADBLOCK_BEHAVIOR = [

tests/test_bd.toml

@@ -4,7 +4,7 @@
 # Note we use 251, a prime, in places to avoid aliasing powers of 2.
 # 
 
-[cases.one_block]
+[cases.test_bd_one_block]
 defines.READ = ['READ_SIZE', 'BLOCK_SIZE']
 defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE']
 code = '''
@@ -29,7 +29,7 @@ code = '''
     }
 '''
 
-[cases.two_block]
+[cases.test_bd_two_block]
 defines.READ = ['READ_SIZE', 'BLOCK_SIZE']
 defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE']
 code = '''
@@ -87,7 +87,7 @@ code = '''
     }
 '''
 
-[cases.last_block]
+[cases.test_bd_last_block]
 defines.READ = ['READ_SIZE', 'BLOCK_SIZE']
 defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE']
 code = '''
@@ -145,7 +145,7 @@ code = '''
     }
 '''
 
-[cases.powers_of_two]
+[cases.test_bd_powers_of_two]
 defines.READ = ['READ_SIZE', 'BLOCK_SIZE']
 defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE']
 code = '''
@@ -191,7 +191,7 @@ code = '''
     }
 '''
 
-[cases.fibonacci]
+[cases.test_bd_fibonacci]
 defines.READ = ['READ_SIZE', 'BLOCK_SIZE']
 defines.PROG = ['PROG_SIZE', 'BLOCK_SIZE']
 code = '''

tests/test_dirs.toml

@@ -1,4 +1,4 @@
-[cases.root]
+[cases.test_dirs_root]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -17,7 +17,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_dir_creation]
+[cases.test_dirs_many_creation]
 defines.N = 'range(3, 100, 3)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -54,7 +54,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_dir_removal]
+[cases.test_dirs_many_removal]
 defines.N = 'range(3, 100, 11)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -111,7 +111,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_dir_rename]
+[cases.test_dirs_many_rename]
 defines.N = 'range(3, 100, 11)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -177,7 +177,7 @@ code = '''
     lfs_unmount(&lfs);
 '''
 
-[cases.reentrant_many_dir]
+[cases.test_dirs_many_reentrant]
 defines.N = [5, 11]
 reentrant = true
 code = '''
@@ -265,7 +265,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.file_creation]
+[cases.test_dirs_file_creation]
 defines.N = 'range(3, 100, 11)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -305,7 +305,7 @@ code = '''
     lfs_unmount(&lfs);
 '''
 
-[cases.file_removal]
+[cases.test_dirs_file_removal]
 defines.N = 'range(3, 100, 11)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -365,7 +365,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.file_rename]
+[cases.test_dirs_file_rename]
 defines.N = 'range(3, 100, 11)'
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -434,7 +434,7 @@ code = '''
     lfs_unmount(&lfs);
 '''
 
-[cases.reentrant_files]
+[cases.test_dirs_file_reentrant]
 defines.N = [5, 25]
 if = 'N < BLOCK_COUNT/2'
 reentrant = true
@@ -524,7 +524,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.nested_dirs]
+[cases.test_dirs_nested]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -652,7 +652,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.recursive_remove]
+[cases.test_dirs_recursive_remove]
 defines.N = [10, 100]
 if = 'N < BLOCK_COUNT/2'
 code = '''
@@ -716,7 +716,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.other_errors]
+[cases.test_dirs_other_errors]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -794,7 +794,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.directory_seek]
+[cases.test_dirs_seek]
 defines.COUNT = [4, 128, 132]
 if = 'COUNT < BLOCK_COUNT/2'
 code = '''
@@ -862,7 +862,7 @@ code = '''
     }
 '''
 
-[cases.root_seek]
+[cases.test_dirs_toot_seek]
 defines.COUNT = [4, 128, 132]
 if = 'COUNT < BLOCK_COUNT/2'
 code = '''

tests/test_entries.toml

@@ -5,7 +5,7 @@
 defines.CACHE_SIZE = 512
 if = 'CACHE_SIZE % PROG_SIZE == 0 && CACHE_SIZE == 512'
 
-[cases.entry_grow]
+[cases.test_entries_grow]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -98,7 +98,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.entry_shrink]
+[cases.test_entries_shrink]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -191,7 +191,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.entry_spill]
+[cases.test_entries_spill]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -268,7 +268,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.entry_push_spill]
+[cases.test_entries_push_spill]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -361,7 +361,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.entry_push_spill_two]
+[cases.test_entries_push_spill_two]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -469,7 +469,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.entry_drop]
+[cases.test_entries_drop]
 code = '''
     uint8_t wbuffer[1024];
     uint8_t rbuffer[1024];
@@ -572,7 +572,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.create_too_big]
+[cases.test_entries_create_too_big]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -600,7 +600,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.resize_too_big]
+[cases.test_entries_resize_too_big]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;

tests/test_evil.toml

@@ -3,7 +3,7 @@
 
 # invalid pointer tests (outside of block_count)
 
-[cases.invalid_tail_pointer]
+[cases.test_evil_invalid_tail_pointer]
 defines.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL']
 defines.INVALSET = [0x3, 0x1, 0x2]
 in = "lfs.c"
@@ -27,7 +27,7 @@ code = '''
     lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT;
 '''
 
-[cases.invalid_dir_pointer]
+[cases.test_evil_invalid_dir_pointer]
 defines.INVALSET = [0x3, 0x1, 0x2]
 in = "lfs.c"
 code = '''
@@ -78,7 +78,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.invalid_file_pointer]
+[cases.test_evil_invalid_file_pointer]
 in = "lfs.c"
 defines.SIZE = [10, 1000, 100000] # faked file size
 code = '''
@@ -130,7 +130,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.invalid_ctz_pointer] # invalid pointer in CTZ skip-list test
+[cases.test_evil_invalid_ctz_pointer] # invalid pointer in CTZ skip-list test
 defines.SIZE = ['2*BLOCK_SIZE', '3*BLOCK_SIZE', '4*BLOCK_SIZE']
 in = "lfs.c"
 code = '''
@@ -196,7 +196,7 @@ code = '''
 '''
 
 
-[cases.invalid_gstate_pointer]
+[cases.test_evil_invalid_gstate_pointer]
 defines.INVALSET = [0x3, 0x1, 0x2]
 in = "lfs.c"
 code = '''
@@ -224,7 +224,7 @@ code = '''
 
 # cycle detection/recovery tests
 
-[cases.mdir_loop] # metadata-pair threaded-list loop test
+[cases.test_evil_mdir_loop] # metadata-pair threaded-list loop test
 in = "lfs.c"
 code = '''
     // create littlefs
@@ -244,7 +244,7 @@ code = '''
     lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT;
 '''
 
-[cases.mdir_loop_2] # metadata-pair threaded-list 2-length loop test
+[cases.test_evil_mdir_loop2] # metadata-pair threaded-list 2-length loop test
 in = "lfs.c"
 code = '''
     // create littlefs with child dir
@@ -275,7 +275,7 @@ code = '''
     lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT;
 '''
 
-[cases.mdir_loop_child] # metadata-pair threaded-list 1-length child loop test
+[cases.test_evil_mdir_loop_child] # metadata-pair threaded-list 1-length child loop test
 in = "lfs.c"
 code = '''
     // create littlefs with child dir

tests/test_exhaustion.toml

@@ -1,5 +1,5 @@
 # test running a filesystem to exhaustion
-[cases.exhaustion]
+[cases.test_exhaustion_normal]
 defines.ERASE_CYCLES = 10
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
@@ -92,7 +92,7 @@ exhausted:
 
 # test running a filesystem to exhaustion
 # which also requires expanding superblocks
-[cases.exhaustion_superblocks]
+[cases.test_exhaustion_superblocks]
 defines.ERASE_CYCLES = 10
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
@@ -186,7 +186,7 @@ exhausted:
 # check for.
 
 # wear-level test running a filesystem to exhaustion
-[cases.wear_leveling_exhaustion]
+[cases.test_exhuastion_wear_leveling]
 defines.ERASE_CYCLES = 20
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
@@ -286,7 +286,7 @@ exhausted:
 '''
 
 # wear-level test + expanding superblock
-[cases.wear_leveling_exhaustion_superblocks]
+[cases.test_exhaustion_wear_leveling_superblocks]
 defines.ERASE_CYCLES = 20
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
@@ -383,7 +383,7 @@ exhausted:
 '''
 
 # test that we wear blocks roughly evenly
-[cases.wear_leveling_distribution]
+[cases.test_exhaustion_wear_distribution]
 defines.ERASE_CYCLES = 0xffffffff
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = [5, 4, 3, 2, 1]

tests/test_files.toml

@@ -1,5 +1,5 @@
 
-[cases.simple_file]
+[cases.test_files_simple]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -22,7 +22,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.large_files]
+[cases.test_files_large]
 defines.SIZE = [32, 8192, 262144, 0, 7, 8193]
 defines.CHUNKSIZE = [31, 16, 33, 1, 1023]
 code = '''
@@ -63,7 +63,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.rewriting_files]
+[cases.test_files_rewrite]
 defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193]
 defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193]
 defines.CHUNKSIZE = [31, 16, 1]
@@ -148,7 +148,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.appending_files]
+[cases.test_files_append]
 defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193]
 defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193]
 defines.CHUNKSIZE = [31, 16, 1]
@@ -228,7 +228,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.truncating_files]
+[cases.test_files_truncate]
 defines.SIZE1 = [32, 8192, 131072, 0, 7, 8193]
 defines.SIZE2 = [32, 8192, 131072, 0, 7, 8193]
 defines.CHUNKSIZE = [31, 16, 1]
@@ -300,7 +300,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.reentrant_file_writing]
+[cases.test_files_reentrant_write]
 defines.SIZE = [32, 0, 7, 2049]
 defines.CHUNKSIZE = [31, 16, 65]
 reentrant = true
@@ -351,7 +351,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.reentrant_file_writing_sync]
+[cases.test_files_reentrant_write_sync]
 defines = [
     # append (O(n))
     {MODE='LFS_O_APPEND',   SIZE=[32, 0, 7, 2049],  CHUNKSIZE=[31, 16, 65]},
@@ -424,7 +424,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_files]
+[cases.test_files_many]
 defines.N = 300
 code = '''
     lfs_t lfs;
@@ -452,7 +452,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_files_power_cycle]
+[cases.test_files_many_power_cycle]
 defines.N = 300
 code = '''
     lfs_t lfs;
@@ -482,7 +482,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.many_files_power_loss]
+[cases.test_files_many_power_loss]
 defines.N = 300
 reentrant = true
 code = '''

tests/test_interspersed.toml

@@ -1,5 +1,5 @@
 
-[cases.interspersed_files]
+[cases.test_interspersed_files]
 defines.SIZE = [10, 100]
 defines.FILES = [4, 10, 26] 
 code = '''
@@ -66,7 +66,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.interspersed_remove_files]
+[cases.test_interspersed_remove_files]
 defines.SIZE = [10, 100]
 defines.FILES = [4, 10, 26]
 code = '''
@@ -127,7 +127,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.remove_inconveniently]
+[cases.test_interspersed_remove_inconveniently]
 defines.SIZE = [10, 100]
 code = '''
     lfs_t lfs;
@@ -191,7 +191,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.reentrant_interspersed_files]
+[cases.test_interspersed_reentrant_files]
 defines.SIZE = [10, 100]
 defines.FILES = [4, 10, 26] 
 reentrant = true

tests/test_move.toml

@@ -1,4 +1,4 @@
-[cases.move_file]
+[cases.test_move_file]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -60,7 +60,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.nop_move] # yes this is legal
+[cases.test_move_nop] # yes this is legal
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -78,7 +78,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_file_corrupt_source]
+[cases.test_move_file_corrupt_source]
 in = "lfs.c"
 code = '''
     lfs_t lfs;
@@ -158,7 +158,7 @@ code = '''
 '''
 
 # move file corrupt source and dest
-[cases.move_file_corrupt_source_dest]
+[cases.test_move_file_corrupt_source_dest]
 in = "lfs.c"
 if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
@@ -254,7 +254,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_file_after_corrupt]
+[cases.test_move_file_after_corrupt]
 in = "lfs.c"
 if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
@@ -355,7 +355,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.reentrant_move_file]
+[cases.test_move_reentrant_file]
 reentrant = true
 code = '''
     lfs_t lfs;
@@ -472,7 +472,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_dir]
+[cases.test_move_dir]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -540,7 +540,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_dir_corrupt_source]
+[cases.test_move_dir_corrupt_source]
 in = "lfs.c"
 code = '''
     lfs_t lfs;
@@ -626,7 +626,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_dir_corrupt_source_dest]
+[cases.test_move_dir_corrupt_source_dest]
 in = "lfs.c"
 if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
@@ -729,7 +729,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_dir_after_corrupt]
+[cases.test_move_dir_after_corrupt]
 in = "lfs.c"
 if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
@@ -837,7 +837,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.reentrant_move_dir]
+[cases.test_reentrant_dir]
 reentrant = true
 code = '''
     lfs_t lfs;
@@ -958,7 +958,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.move_state_stealing]
+[cases.test_move_state_stealing]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -1028,7 +1028,7 @@ code = '''
 # Other specific corner cases
 
 # create + delete in same commit with neighbors
-[cases.create_delete_same]
+[cases.test_move_create_delete_same]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -1179,7 +1179,7 @@ code = '''
 '''
 
 # create + delete + delete in same commit with neighbors
-[cases.create_delete_delete_same]
+[cases.test_move_create_delete_delete_same]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -1341,7 +1341,7 @@ code = '''
 '''
 
 # create + delete in different dirs with neighbors
-[cases.create_delete_different]
+[cases.test_move_create_delete_different]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -1584,7 +1584,7 @@ code = '''
 '''
 
 # move fix in relocation
-[cases.move_fix_relocation]
+[cases.test_move_fix_relocation]
 in = "lfs.c"
 defines.RELOCATIONS = 'range(4)'
 defines.ERASE_CYCLES = 0xffffffff
@@ -1729,7 +1729,7 @@ code = '''
 '''
 
 # move fix in relocation with predecessor
-[cases.move_fix_relocation_predecessor]
+[cases.test_move_fix_relocation_predecessor]
 in = "lfs.c"
 defines.RELOCATIONS = 'range(8)'
 defines.ERASE_CYCLES = 0xffffffff

tests/test_orphans.toml

@@ -1,4 +1,4 @@
-[cases.orphan]
+[cases.test_orphans_normal]
 in = "lfs.c"
 if = 'PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
@@ -60,7 +60,7 @@ code = '''
 '''
 
 # reentrant testing for orphans, basically just spam mkdir/remove
-[cases.reentrant_orphan]
+[cases.test_orphans_reentrant]
 reentrant = true
 # TODO fix this case, caused by non-DAG trees
 if = '!(DEPTH == 3 && CACHE_SIZE != 64)'

tests/test_paths.toml

@@ -1,6 +1,6 @@
 
 # simple path test
-[cases.path]
+[cases.test_paths_normal]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -25,7 +25,7 @@ code = '''
 '''
 
 # redundant slashes
-[cases.redundant_slashes]
+[cases.test_paths_redundant_slashes]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -52,7 +52,7 @@ code = '''
 '''
 
 # dot path test
-[cases.dot_path]
+[cases.test_paths_dot]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -81,7 +81,7 @@ code = '''
 '''
 
 # dot dot path test
-[cases.dot_dot_path]
+[cases.test_paths_dot_dot]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -114,7 +114,7 @@ code = '''
 '''
 
 # trailing dot path test
-[cases.trailing_dot_path]
+[cases.test_paths_trailing_dot]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -139,7 +139,7 @@ code = '''
 '''
 
 # leading dot path test
-[cases.leading_dot_path]
+[cases.test_paths_leading_dot]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -154,7 +154,7 @@ code = '''
 '''
 
 # root dot dot path test
-[cases.root_dot_dot_path]
+[cases.test_paths_root_dot_dot]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -181,7 +181,7 @@ code = '''
 '''
 
 # invalid path tests
-[cases.invalid_path]
+[cases.test_paths_invalid]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg);
@@ -206,7 +206,7 @@ code = '''
 '''
 
 # root operations
-[cases.root]
+[cases.test_paths_root]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -226,7 +226,7 @@ code = '''
 '''
 
 # root representations
-[cases.root_reprs]
+[cases.test_paths_root_reprs]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -254,7 +254,7 @@ code = '''
 '''
 
 # superblock conflict test
-[cases.superblock_conflict]
+[cases.test_paths_superblock_conflict]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -273,7 +273,7 @@ code = '''
 '''
 
 # max path test
-[cases.max_path]
+[cases.test_paths_max]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -301,7 +301,7 @@ code = '''
 '''
 
 # really big path test
-[cases.really_big_path]
+[cases.test_paths_really_big]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;

tests/test_relocations.toml

@@ -1,5 +1,5 @@
 # specific corner cases worth explicitly testing for
-[cases.dangling_split_dir]
+[cases.test_relocations_dangling_split_dir]
 defines.ITERATIONS = 20
 defines.COUNT = 10
 defines.BLOCK_CYCLES = [8, 1]
@@ -77,7 +77,7 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
-[cases.outdated_head]
+[cases.test_relocations_outdated_head]
 defines.ITERATIONS = 20
 defines.COUNT = 10
 defines.BLOCK_CYCLES = [8, 1]
@@ -168,7 +168,7 @@ code = '''
 # reentrant testing for relocations, this is the same as the
 # orphan testing, except here we also set block_cycles so that
 # almost every tree operation needs a relocation
-[cases.reentrant_relocations]
+[cases.test_relocations_reentrant]
 reentrant = true
 # TODO fix this case, caused by non-DAG trees
 if = '!(DEPTH == 3 && CACHE_SIZE != 64)'
@@ -236,7 +236,7 @@ code = '''
 '''
 
 # reentrant testing for relocations, but now with random renames!
-[cases.reentrant_relocations_renames]
+[cases.test_relocations_reentrant_renames]
 reentrant = true
 # TODO fix this case, caused by non-DAG trees
 if = '!(DEPTH == 3 && CACHE_SIZE != 64)'

tests/test_seek.toml

@@ -1,6 +1,6 @@
 
 # simple file seek
-[cases.seek]
+[cases.test_seek_read]
 defines = [
     {COUNT=132, SKIP=4},
     {COUNT=132, SKIP=128},
@@ -73,7 +73,7 @@ code = '''
 '''
 
 # simple file seek and write
-[cases.seek_write]
+[cases.test_seek_write]
 defines = [
     {COUNT=132, SKIP=4},
     {COUNT=132, SKIP=128},
@@ -138,7 +138,7 @@ code = '''
 '''
 
 # boundary seek and writes
-[cases.boundary_seek_write]
+[cases.test_seek_boundary_write]
 defines.COUNT = 132
 code = '''
     lfs_t lfs;
@@ -195,7 +195,7 @@ code = '''
 '''
 
 # out of bounds seek
-[cases.out_of_bounds_seek]
+[cases.test_seek_out_of_bounds]
 defines = [
     {COUNT=132, SKIP=4},
     {COUNT=132, SKIP=128},
@@ -254,7 +254,7 @@ code = '''
 '''
 
 # inline write and seek
-[cases.inline_write_seek]
+[cases.test_seek_inline_write]
 defines.SIZE = [2, 4, 128, 132]
 code = '''
     lfs_t lfs;
@@ -325,7 +325,7 @@ code = '''
 '''
 
 # file seek and write with power-loss
-[cases.reentrant_seek_write]
+[cases.test_seek_reentrant_write]
 # must be power-of-2 for quadratic probing to be exhaustive
 defines.COUNT = [4, 64, 128]
 reentrant = true

tests/test_superblocks.toml

@@ -1,12 +1,12 @@
 # simple formatting test
-[cases.format]
+[cases.test_superblocks_format]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
 '''
 
 # mount/unmount
-[cases.mount]
+[cases.test_superblocks_mount]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -15,7 +15,7 @@ code = '''
 '''
 
 # reentrant format
-[cases.reentrant_format]
+[cases.test_superblocks_reentrant_format]
 reentrant = true
 code = '''
     lfs_t lfs;
@@ -28,14 +28,14 @@ code = '''
 '''
 
 # invalid mount
-[cases.invalid_mount]
+[cases.test_superblocks_invalid_mount]
 code = '''
     lfs_t lfs;
     lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT;
 '''
 
 # expanding superblock
-[cases.expanding_superblock]
+[cases.test_superblocks_expand]
 defines.LFS_BLOCK_CYCLES = [32, 33, 1]
 defines.N = [10, 100, 1000]
 code = '''
@@ -69,7 +69,7 @@ code = '''
 '''
 
 # expanding superblock with power cycle
-[cases.expanding_superblock_power_cycle]
+[cases.test_superblocks_expand_power_cycle]
 defines.LFS_BLOCK_CYCLES = [32, 33, 1]
 defines.N = [10, 100, 1000]
 code = '''
@@ -107,7 +107,7 @@ code = '''
 '''
 
 # reentrant expanding superblock
-[cases.reentrant_expanding_superblock]
+[cases.test_superblocks_reentrant_expand]
 defines.LFS_BLOCK_CYCLES = [2, 1]
 defines.N = 24
 reentrant = true

tests/test_truncate.toml

@@ -1,5 +1,5 @@
 # simple truncate
-[cases.truncate]
+[cases.test_truncate_simple]
 defines.MEDIUMSIZE = [32, 2048]
 defines.LARGESIZE = 8192
 code = '''
@@ -47,7 +47,7 @@ code = '''
 '''
 
 # truncate and read
-[cases.truncate_read]
+[cases.test_truncate_read]
 defines.MEDIUMSIZE = [32, 2048]
 defines.LARGESIZE = 8192
 code = '''
@@ -102,7 +102,7 @@ code = '''
 '''
 
 # write, truncate, and read
-[cases.write_truncate_read]
+[cases.test_truncate_write_read]
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -158,7 +158,7 @@ code = '''
 '''
 
 # truncate and write
-[cases.truncate_write]
+[cases.test_truncate_write]
 defines.MEDIUMSIZE = [32, 2048]
 defines.LARGESIZE = 8192
 code = '''
@@ -213,7 +213,7 @@ code = '''
 '''
 
 # truncate write under powerloss
-[cases.reentrant_truncate_write]
+[cases.test_truncate_reentrant_write]
 defines.SMALLSIZE = [4, 512]
 defines.MEDIUMSIZE = [32, 1024]
 defines.LARGESIZE = 2048
@@ -284,7 +284,7 @@ code = '''
 '''
 
 # more aggressive general truncation tests
-[cases.aggressive_truncate]
+[cases.test_truncate_aggressive]
 defines.CONFIG = 'range(6)'
 defines.SMALLSIZE = 32
 defines.MEDIUMSIZE = 2048
@@ -428,7 +428,7 @@ code = '''
 '''
 
 # noop truncate
-[cases.nop_truncate]
+[cases.test_truncate_nop]
 defines.MEDIUMSIZE = [32, 2048]
 code = '''
     lfs_t lfs;