Added bench.py and bench_runner.c for benchmarking

These are really just different flavors of test.py and test_runner.c
without support for power-loss testing, but with support for measuring
the cumulative number of bytes read, programmed, and erased.

Note that the existing define parameterization should work perfectly
fine for running benchmarks across various dimensions:

./scripts/bench.py \
    runners/bench_runner \
    bench_file_read \
    -gnor \
    -DSIZE='range(0,131072,1024)'

Also added a couple basic benchmarks as a starting point.

.gitignore

@@ -5,6 +5,7 @@
 *.ci
 *.csv
 *.t.c
+*.b.c
 *.a.c
 *.gcno
 *.gcda
@@ -17,3 +18,4 @@ tests/*.toml.*
 scripts/__pycache__
 .gdb_history
 runners/test_runner
+runners/bench_runner

Makefile

@@ -7,7 +7,8 @@ $(if $(findstring n,$(MAKEFLAGS)),, $(shell mkdir -p \
 	$(BUILDDIR) \
 	$(BUILDDIR)bd \
 	$(BUILDDIR)runners \
-	$(BUILDDIR)tests))
+	$(BUILDDIR)tests \
+	$(BUILDDIR)benches))
 endif
 
 # overridable target/src/tools/flags/etc
@@ -45,6 +46,18 @@ TEST_CI	:= $(TEST_TAC:%.t.a.c=%.t.a.ci)
 TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno)
 TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda)
 
+BENCHES ?= $(wildcard benches/*.toml)
+BENCH_SRC ?= $(SRC) \
+		$(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \
+		runners/bench_runner.c
+BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c)
+BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c)
+BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o)
+BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d)
+BENCH_CI	:= $(BENCH_BAC:%.b.a.c=%.b.a.ci)
+BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno)
+BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda)
+
 ifdef DEBUG
 override CFLAGS += -O0
 else
@@ -60,27 +73,31 @@ override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
 override CFLAGS += -ftrack-macro-expansion=0
 
 override TESTFLAGS += -b
+override BENCHFLAGS += -b
 # forward -j flag
 override TESTFLAGS += $(filter -j%,$(MAKEFLAGS))
+override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS))
 ifdef VERBOSE
-override TESTFLAGS     += -v
-override CODEFLAGS     += -v
-override DATAFLAGS     += -v
-override STACKFLAGS    += -v
-override STRUCTFLAGS   += -v
-override COVERAGEFLAGS += -v
-override TESTFLAGS     += -v
-override TESTCFLAGS    += -v
+override CODEFLAGS     	+= -v
+override DATAFLAGS     	+= -v
+override STACKFLAGS    	+= -v
+override STRUCTFLAGS   	+= -v
+override COVERAGEFLAGS 	+= -v
+override TESTFLAGS     	+= -v
+override TESTCFLAGS    	+= -v
+override BENCHFLAGS    	+= -v
+override BENCHCFLAGS  	+= -v
 endif
 ifdef EXEC
-override TESTFLAGS 	   += --exec="$(EXEC)"
+override TESTFLAGS 	   	+= --exec="$(EXEC)"
+override BENCHFLAGS	   	+= --exec="$(EXEC)"
 endif
 ifdef BUILDDIR
-override CODEFLAGS     += --build-dir="$(BUILDDIR:/=)"
-override DATAFLAGS     += --build-dir="$(BUILDDIR:/=)"
-override STACKFLAGS    += --build-dir="$(BUILDDIR:/=)"
-override STRUCTFLAGS   += --build-dir="$(BUILDDIR:/=)"
-override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)"
+override CODEFLAGS     	+= --build-dir="$(BUILDDIR:/=)"
+override DATAFLAGS     	+= --build-dir="$(BUILDDIR:/=)"
+override STACKFLAGS    	+= --build-dir="$(BUILDDIR:/=)"
+override STRUCTFLAGS   	+= --build-dir="$(BUILDDIR:/=)"
+override COVERAGEFLAGS	+= --build-dir="$(BUILDDIR:/=)"
 endif
 ifneq ($(NM),nm)
 override CODEFLAGS += --nm-tool="$(NM)"
@@ -119,6 +136,17 @@ test: test-runner
 test-list: test-runner
 	./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l
 
+.PHONY: bench-runner build-bench
+bench-runner build-bench: $(BUILDDIR)runners/bench_runner
+
+.PHONY: bench
+bench: bench-runner
+	./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS)
+
+.PHONY: bench-list
+bench-list: bench-runner
+	./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) -l
+
 .PHONY: code
 code: $(OBJ)
 	./scripts/code.py $^ -S $(CODEFLAGS)
@@ -186,6 +214,9 @@ $(BUILDDIR)lfs.csv: \
 $(BUILDDIR)runners/test_runner: $(TEST_OBJ)
 	$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
 
+$(BUILDDIR)runners/bench_runner: $(BENCH_OBJ)
+	$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
+
 # our main build rule generates .o, .d, and .ci files, the latter
 # used for stack analysis
 $(BUILDDIR)%.o $(BUILDDIR)%.ci: %.c
@@ -206,6 +237,12 @@ $(BUILDDIR)%.t.c: %.toml
 $(BUILDDIR)%.t.c: %.c $(TESTS)
 	./scripts/test.py -c $(TESTS) -s $< $(TESTCFLAGS) -o $@
 
+$(BUILDDIR)%.b.c: %.toml
+	./scripts/bench.py -c $< $(BENCHCFLAGS) -o $@
+
+$(BUILDDIR)%.b.c: %.c $(BENCHES)
+	./scripts/bench.py -c $(BENCHES) -s $< $(BENCHCFLAGS) -o $@
+
 # clean everything
 .PHONY: clean
 clean:
@@ -219,6 +256,7 @@ clean:
 		$(BUILDDIR)lfs.struct.csv \
 		$(BUILDDIR)lfs.coverage.csv)
 	rm -f $(BUILDDIR)runners/test_runner
+	rm -f $(BUILDDIR)runners/bench_runner
 	rm -f $(OBJ)
 	rm -f $(DEP)
 	rm -f $(ASM)
@@ -230,3 +268,10 @@ clean:
 	rm -f $(TEST_CI)
 	rm -f $(TEST_GCNO)
 	rm -f $(TEST_GCDA)
+	rm -f $(BENCH_BC)
+	rm -f $(BENCH_BAC)
+	rm -f $(BENCH_OBJ)
+	rm -f $(BENCH_DEP)
+	rm -f $(BENCH_CI)
+	rm -f $(BENCH_GCNO)
+	rm -f $(BENCH_GCDA)

bd/lfs_testbd.c → bd/lfs_emubd.c

@@ -1,5 +1,5 @@
 /*
- * Testing block device, wraps filebd and rambd while providing a bunch
+ * Emulating block device, wraps filebd and rambd while providing a bunch
  * of hooks for testing littlefs in various conditions.
  *
  * Copyright (c) 2022, The littlefs authors.
@@ -11,7 +11,7 @@
 #define _POSIX_C_SOURCE 199309L
 #endif
 
-#include "bd/lfs_testbd.h"
+#include "bd/lfs_emubd.h"
 
 #include <stdlib.h>
 #include <fcntl.h>
@@ -29,14 +29,14 @@
 // Note we can only modify a block if we have exclusive access to it (rc == 1)
 //
 
-static lfs_testbd_block_t *lfs_testbd_incblock(lfs_testbd_block_t *block) {
+static lfs_emubd_block_t *lfs_emubd_incblock(lfs_emubd_block_t *block) {
     if (block) {
         block->rc += 1;
     }
     return block;
 }
 
-static void lfs_testbd_decblock(lfs_testbd_block_t *block) {
+static void lfs_emubd_decblock(lfs_emubd_block_t *block) {
     if (block) {
         block->rc -= 1;
         if (block->rc == 0) {
@@ -45,34 +45,34 @@ static void lfs_testbd_decblock(lfs_testbd_block_t *block) {
     }
 }
 
-static lfs_testbd_block_t *lfs_testbd_mutblock(
+static lfs_emubd_block_t *lfs_emubd_mutblock(
         const struct lfs_config *cfg,
-        lfs_testbd_block_t **block) {
-    lfs_testbd_block_t *block_ = *block;
+        lfs_emubd_block_t **block) {
+    lfs_emubd_block_t *block_ = *block;
     if (block_ && block_->rc == 1) {
         // rc == 1? can modify
         return block_;
 
     } else if (block_) {
         // rc > 1? need to create a copy
-        lfs_testbd_block_t *nblock = malloc(
-                sizeof(lfs_testbd_block_t) + cfg->block_size);
+        lfs_emubd_block_t *nblock = malloc(
+                sizeof(lfs_emubd_block_t) + cfg->block_size);
         if (!nblock) {
             return NULL;
         }
 
         memcpy(nblock, block_,
-                sizeof(lfs_testbd_block_t) + cfg->block_size);
+                sizeof(lfs_emubd_block_t) + cfg->block_size);
         nblock->rc = 1;
 
-        lfs_testbd_decblock(block_);
+        lfs_emubd_decblock(block_);
         *block = nblock;
         return nblock;
 
     } else {
         // no block? need to allocate
-        lfs_testbd_block_t *nblock = malloc(
-                sizeof(lfs_testbd_block_t) + cfg->block_size);
+        lfs_emubd_block_t *nblock = malloc(
+                sizeof(lfs_emubd_block_t) + cfg->block_size);
         if (!nblock) {
             return NULL;
         }
@@ -81,7 +81,7 @@ static lfs_testbd_block_t *lfs_testbd_mutblock(
         nblock->wear = 0;
 
         // zero for consistency
-        lfs_testbd_t *bd = cfg->context;
+        lfs_emubd_t *bd = cfg->context;
         memset(nblock->data,
                 (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0,
                 cfg->block_size);
@@ -92,11 +92,11 @@ static lfs_testbd_block_t *lfs_testbd_mutblock(
 }
 
 
-// testbd create/destroy
+// emubd create/destroy
 
-int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
-        const struct lfs_testbd_config *bdcfg) {
-    LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, "
+int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path,
+        const struct lfs_emubd_config *bdcfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -113,25 +113,28 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
             bdcfg->badblock_behavior, bdcfg->power_cycles,
             bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb,
             bdcfg->powerloss_data, bdcfg->track_branches);
-    lfs_testbd_t *bd = cfg->context;
+    lfs_emubd_t *bd = cfg->context;
     bd->cfg = bdcfg;
 
     // allocate our block array, all blocks start as uninitialized
-    bd->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*));
+    bd->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*));
     if (!bd->blocks) {
-        LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
-    memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_testbd_block_t*));
+    memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*));
 
     // setup testing things
+    bd->read = 0;
+    bd->prog = 0;
+    bd->erased = 0;
     bd->power_cycles = bd->cfg->power_cycles;
     bd->disk = NULL;
 
     if (bd->cfg->disk_path) {
-        bd->disk = malloc(sizeof(lfs_testbd_disk_t));
+        bd->disk = malloc(sizeof(lfs_emubd_disk_t));
         if (!bd->disk) {
-            LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
+            LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM);
             return LFS_ERR_NOMEM;
         }
         bd->disk->rc = 1;
@@ -146,7 +149,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         #endif
         if (bd->disk->fd < 0) {
             int err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err);
             return err;
         }
 
@@ -155,7 +158,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         if (bd->cfg->erase_value != -1) {
             bd->disk->scratch = malloc(cfg->block_size);
             if (!bd->disk->scratch) {
-                LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
+                LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM);
                 return LFS_ERR_NOMEM;
             }
             memset(bd->disk->scratch,
@@ -164,12 +167,12 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         }
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", 0);
     return 0;
 }
 
-int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
-    LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, "
+int lfs_emubd_create(const struct lfs_config *cfg, const char *path) {
+    LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -179,19 +182,19 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
             (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync,
             cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count,
             path);
-    static const struct lfs_testbd_config defaults = {.erase_value=-1};
-    int err = lfs_testbd_createcfg(cfg, path, &defaults);
-    LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err);
+    static const struct lfs_emubd_config defaults = {.erase_value=-1};
+    int err = lfs_emubd_createcfg(cfg, path, &defaults);
+    LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err);
     return err;
 }
 
-int lfs_testbd_destroy(const struct lfs_config *cfg) {
-    LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg);
-    lfs_testbd_t *bd = cfg->context;
+int lfs_emubd_destroy(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
 
     // decrement reference counts
     for (lfs_block_t i = 0; i < cfg->block_count; i++) {
-        lfs_testbd_decblock(bd->blocks[i]);
+        lfs_emubd_decblock(bd->blocks[i]);
     }
     free(bd->blocks);
 
@@ -205,7 +208,7 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) {
         }
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_destroy -> %d", 0);
     return 0;
 }
 
@@ -213,12 +216,12 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) {
 
 // block device API
 
-int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
+int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TESTBD_TRACE("lfs_testbd_read(%p, "
+    LFS_EMUBD_TRACE("lfs_emubd_read(%p, "
                 "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
-    lfs_testbd_t *bd = cfg->context;
+    lfs_emubd_t *bd = cfg->context;
 
     // check if read is valid
     LFS_ASSERT(block < cfg->block_count);
@@ -227,12 +230,12 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(off+size <= cfg->block_size);
 
     // get the block
-    const lfs_testbd_block_t *b = bd->blocks[block];
+    const lfs_emubd_block_t *b = bd->blocks[block];
     if (b) {
         // block bad?
         if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles &&
-                bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) {
-            LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT);
+                bd->cfg->badblock_behavior == LFS_EMUBD_BADBLOCK_READERROR) {
+            LFS_EMUBD_TRACE("lfs_emubd_read -> %d", LFS_ERR_CORRUPT);
             return LFS_ERR_CORRUPT;
         }
 
@@ -243,8 +246,10 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
         memset(buffer,
                 (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0,
                 size);
-    }
+    }   
 
+    // track reads
+    bd->read += size;
     if (bd->cfg->read_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->read_sleep/1000000000,
@@ -252,21 +257,21 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
             NULL);
         if (err) {
             err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_read -> %d", err);
             return err;
         }
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_read -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_read -> %d", 0);
     return 0;
 }
 
-int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
+int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TESTBD_TRACE("lfs_testbd_prog(%p, "
+    LFS_EMUBD_TRACE("lfs_emubd_prog(%p, "
                 "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
-    lfs_testbd_t *bd = cfg->context;
+    lfs_emubd_t *bd = cfg->context;
 
     // check if write is valid
     LFS_ASSERT(block < cfg->block_count);
@@ -275,23 +280,23 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(off+size <= cfg->block_size);
 
     // get the block
-    lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]);
+    lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]);
     if (!b) {
-        LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
 
     // block bad?
     if (bd->cfg->erase_cycles && b->wear >= bd->cfg->erase_cycles) {
         if (bd->cfg->badblock_behavior ==
-                LFS_TESTBD_BADBLOCK_PROGERROR) {
-            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT);
+                LFS_EMUBD_BADBLOCK_PROGERROR) {
+            LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_CORRUPT);
             return LFS_ERR_CORRUPT;
         } else if (bd->cfg->badblock_behavior ==
-                LFS_TESTBD_BADBLOCK_PROGNOOP ||
+                LFS_EMUBD_BADBLOCK_PROGNOOP ||
                 bd->cfg->badblock_behavior ==
-                LFS_TESTBD_BADBLOCK_ERASENOOP) {
-            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
+                LFS_EMUBD_BADBLOCK_ERASENOOP) {
+            LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0);
             return 0;
         }
     }
@@ -313,18 +318,20 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
                 SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err);
             return err;
         }
 
         ssize_t res2 = write(bd->disk->fd, buffer, size);
         if (res2 < 0) {
             int err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err);
             return err;
         }
     }
 
+    // track progs
+    bd->prog += size;
     if (bd->cfg->prog_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->prog_sleep/1000000000,
@@ -332,7 +339,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
             NULL);
         if (err) {
             err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", err);
             return err;
         }
     }
@@ -346,21 +353,21 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         }
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0);
     return 0;
 }
 
-int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
-    lfs_testbd_t *bd = cfg->context;
+int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) {
+    LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    lfs_emubd_t *bd = cfg->context;
 
     // check if erase is valid
     LFS_ASSERT(block < cfg->block_count);
 
     // get the block
-    lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]);
+    lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]);
     if (!b) {
-        LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
 
@@ -368,12 +375,12 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
     if (bd->cfg->erase_cycles) {
         if (b->wear >= bd->cfg->erase_cycles) {
             if (bd->cfg->badblock_behavior ==
-                    LFS_TESTBD_BADBLOCK_ERASEERROR) {
-                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT);
+                    LFS_EMUBD_BADBLOCK_ERASEERROR) {
+                LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", LFS_ERR_CORRUPT);
                 return LFS_ERR_CORRUPT;
             } else if (bd->cfg->badblock_behavior ==
-                    LFS_TESTBD_BADBLOCK_ERASENOOP) {
-                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0);
+                    LFS_EMUBD_BADBLOCK_ERASENOOP) {
+                LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", 0);
                 return 0;
             }
         } else {
@@ -393,7 +400,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
                     SEEK_SET);
             if (res1 < 0) {
                 int err = -errno;
-                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err);
+                LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err);
                 return err;
             }
 
@@ -402,12 +409,14 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
                     cfg->block_size);
             if (res2 < 0) {
                 int err = -errno;
-                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err);
+                LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err);
                 return err;
             }
         }
     }
 
+    // track erases
+    bd->erased += cfg->block_size;
     if (bd->cfg->erase_sleep) {
         int err = nanosleep(&(struct timespec){
                 .tv_sec=bd->cfg->erase_sleep/1000000000,
@@ -415,7 +424,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
             NULL);
         if (err) {
             err = -errno;
-            LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err);
+            LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err);
             return err;
         }
     }
@@ -429,98 +438,142 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         }
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_prog -> %d", 0);
     return 0;
 }
 
-int lfs_testbd_sync(const struct lfs_config *cfg) {
-    LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg);
+int lfs_emubd_sync(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_sync(%p)", (void*)cfg);
 
     // do nothing
     (void)cfg;
 
-    LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_sync -> %d", 0);
     return 0;
 }
 
+/// Additional extended API for driving test features ///
+
+lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_getread(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
+    LFS_EMUBD_TRACE("lfs_emubd_getread -> %"PRIu64, bd->read);
+    return bd->read;
+}
+
+lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_getprog(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
+    LFS_EMUBD_TRACE("lfs_emubd_getprog -> %"PRIu64, bd->prog);
+    return bd->prog;
+}
+
+lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg) {
+    LFS_EMUBD_TRACE("lfs_emubd_geterased(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
+    LFS_EMUBD_TRACE("lfs_emubd_geterased -> %"PRIu64, bd->erased);
+    return bd->erased;
+}
 
-// simulated wear operations
+int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read) {
+    LFS_EMUBD_TRACE("lfs_emubd_setread(%p, %"PRIu64")", (void*)cfg, read);
+    lfs_emubd_t *bd = cfg->context;
+    bd->read = read;
+    LFS_EMUBD_TRACE("lfs_emubd_setread -> %d", 0);
+    return 0;
+}
+
+int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog) {
+    LFS_EMUBD_TRACE("lfs_emubd_setprog(%p, %"PRIu64")", (void*)cfg, prog);
+    lfs_emubd_t *bd = cfg->context;
+    bd->prog = prog;
+    LFS_EMUBD_TRACE("lfs_emubd_setprog -> %d", 0);
+    return 0;
+}
+
+int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased) {
+    LFS_EMUBD_TRACE("lfs_emubd_seterased(%p, %"PRIu64")", (void*)cfg, erased);
+    lfs_emubd_t *bd = cfg->context;
+    bd->erased = erased;
+    LFS_EMUBD_TRACE("lfs_emubd_seterased -> %d", 0);
+    return 0;
+}
 
-lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg,
+lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg,
         lfs_block_t block) {
-    LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block);
-    lfs_testbd_t *bd = cfg->context;
+    LFS_EMUBD_TRACE("lfs_emubd_getwear(%p, %"PRIu32")", (void*)cfg, block);
+    lfs_emubd_t *bd = cfg->context;
 
     // check if block is valid
     LFS_ASSERT(block < cfg->block_count);
 
     // get the wear
-    lfs_testbd_wear_t wear;
-    const lfs_testbd_block_t *b = bd->blocks[block];
+    lfs_emubd_wear_t wear;
+    const lfs_emubd_block_t *b = bd->blocks[block];
     if (b) {
         wear = b->wear;
     } else {
         wear = 0;
     }
 
-    LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, wear);
+    LFS_EMUBD_TRACE("lfs_emubd_getwear -> %"PRIu32, wear);
     return wear;
 }
 
-int lfs_testbd_setwear(const struct lfs_config *cfg,
-        lfs_block_t block, lfs_testbd_wear_t wear) {
-    LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block);
-    lfs_testbd_t *bd = cfg->context;
+int lfs_emubd_setwear(const struct lfs_config *cfg,
+        lfs_block_t block, lfs_emubd_wear_t wear) {
+    LFS_EMUBD_TRACE("lfs_emubd_setwear(%p, %"PRIu32")", (void*)cfg, block);
+    lfs_emubd_t *bd = cfg->context;
 
     // check if block is valid
     LFS_ASSERT(block < cfg->block_count);
 
     // set the wear
-    lfs_testbd_block_t *b = lfs_testbd_mutblock(cfg, &bd->blocks[block]);
+    lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]);
     if (!b) {
-        LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
     b->wear = wear;
 
-    LFS_TESTBD_TRACE("lfs_testbd_setwear -> %"PRIu32, 0);
+    LFS_EMUBD_TRACE("lfs_emubd_setwear -> %"PRIu32, 0);
     return 0;
 }
 
-lfs_testbd_spowercycles_t lfs_testbd_getpowercycles(
+lfs_emubd_spowercycles_t lfs_emubd_getpowercycles(
         const struct lfs_config *cfg) {
-    LFS_TESTBD_TRACE("lfs_testbd_getpowercycles(%p)", (void*)cfg);
-    lfs_testbd_t *bd = cfg->context;
+    LFS_EMUBD_TRACE("lfs_emubd_getpowercycles(%p)", (void*)cfg);
+    lfs_emubd_t *bd = cfg->context;
 
-    LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %"PRIi32, bd->power_cycles);
+    LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %"PRIi32, bd->power_cycles);
     return bd->power_cycles;
 }
 
-int lfs_testbd_setpowercycles(const struct lfs_config *cfg,
-        lfs_testbd_powercycles_t power_cycles) {
-    LFS_TESTBD_TRACE("lfs_testbd_setpowercycles(%p, %"PRIi32")",
+int lfs_emubd_setpowercycles(const struct lfs_config *cfg,
+        lfs_emubd_powercycles_t power_cycles) {
+    LFS_EMUBD_TRACE("lfs_emubd_setpowercycles(%p, %"PRIi32")",
             (void*)cfg, power_cycles);
-    lfs_testbd_t *bd = cfg->context;
+    lfs_emubd_t *bd = cfg->context;
 
     bd->power_cycles = power_cycles;
 
-    LFS_TESTBD_TRACE("lfs_testbd_getpowercycles -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_getpowercycles -> %d", 0);
     return 0;
 }
 
-int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) {
-    LFS_TESTBD_TRACE("lfs_testbd_copy(%p, %p)", (void*)cfg, (void*)copy);
-    lfs_testbd_t *bd = cfg->context;
+int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) {
+    LFS_EMUBD_TRACE("lfs_emubd_copy(%p, %p)", (void*)cfg, (void*)copy);
+    lfs_emubd_t *bd = cfg->context;
 
     // lazily copy over our block array
-    copy->blocks = malloc(cfg->block_count * sizeof(lfs_testbd_block_t*));
+    copy->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*));
     if (!copy->blocks) {
-        LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", LFS_ERR_NOMEM);
+        LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM);
         return LFS_ERR_NOMEM;
     }
 
     for (size_t i = 0; i < cfg->block_count; i++) {
-        copy->blocks[i] = lfs_testbd_incblock(bd->blocks[i]);
+        copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]);
     }
 
     // other state
@@ -531,7 +584,7 @@ int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy) {
     }
     copy->cfg = bd->cfg;
 
-    LFS_TESTBD_TRACE("lfs_testbd_copy -> %d", 0);
+    LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", 0);
     return 0;
 }
 

bd/lfs_testbd.h → bd/lfs_emubd.h

@@ -1,13 +1,13 @@
 /*
- * Testing block device, wraps filebd and rambd while providing a bunch
+ * Emulating block device, wraps filebd and rambd while providing a bunch
  * of hooks for testing littlefs in various conditions.
  *
  * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
-#ifndef LFS_TESTBD_H
-#define LFS_TESTBD_H
+#ifndef LFS_EMUBD_H
+#define LFS_EMUBD_H
 
 #include "lfs.h"
 #include "lfs_util.h"
@@ -21,11 +21,11 @@ extern "C"
 
 
 // Block device specific tracing
-#ifndef LFS_TESTBD_TRACE
-#ifdef LFS_TESTBD_YES_TRACE
-#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#ifndef LFS_EMUBD_TRACE
+#ifdef LFS_EMUBD_YES_TRACE
+#define LFS_EMUBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
 #else
-#define LFS_TESTBD_TRACE(...)
+#define LFS_EMUBD_TRACE(...)
 #endif
 #endif
 
@@ -35,34 +35,38 @@ extern "C"
 //
 // Not that read-noop is not allowed. Read _must_ return a consistent (but
 // may be arbitrary) value on every read.
-typedef enum lfs_testbd_badblock_behavior {
-    LFS_TESTBD_BADBLOCK_PROGERROR,
-    LFS_TESTBD_BADBLOCK_ERASEERROR,
-    LFS_TESTBD_BADBLOCK_READERROR,
-    LFS_TESTBD_BADBLOCK_PROGNOOP,
-    LFS_TESTBD_BADBLOCK_ERASENOOP,
-} lfs_testbd_badblock_behavior_t;
+typedef enum lfs_emubd_badblock_behavior {
+    LFS_EMUBD_BADBLOCK_PROGERROR,
+    LFS_EMUBD_BADBLOCK_ERASEERROR,
+    LFS_EMUBD_BADBLOCK_READERROR,
+    LFS_EMUBD_BADBLOCK_PROGNOOP,
+    LFS_EMUBD_BADBLOCK_ERASENOOP,
+} lfs_emubd_badblock_behavior_t;
 
 // Mode determining how power-loss behaves during testing. For now this
 // only supports a noop behavior, leaving the data on-disk untouched.
-typedef enum lfs_testbd_powerloss_behavior {
-    LFS_TESTBD_POWERLOSS_NOOP,
-} lfs_testbd_powerloss_behavior_t;
+typedef enum lfs_emubd_powerloss_behavior {
+    LFS_EMUBD_POWERLOSS_NOOP,
+} lfs_emubd_powerloss_behavior_t;
+
+// Type for measuring read/program/erase operations
+typedef uint64_t lfs_emubd_io_t;
+typedef int64_t lfs_emubd_sio_t;
 
 // Type for measuring wear
-typedef uint32_t lfs_testbd_wear_t;
-typedef int32_t lfs_testbd_swear_t;
+typedef uint32_t lfs_emubd_wear_t;
+typedef int32_t lfs_emubd_swear_t;
 
 // Type for tracking power-cycles
-typedef uint32_t lfs_testbd_powercycles_t;
-typedef int32_t lfs_testbd_spowercycles_t;
+typedef uint32_t lfs_emubd_powercycles_t;
+typedef int32_t lfs_emubd_spowercycles_t;
 
 // Type for delays in nanoseconds
-typedef uint64_t lfs_testbd_sleep_t;
-typedef int64_t lfs_testbd_ssleep_t;
+typedef uint64_t lfs_emubd_sleep_t;
+typedef int64_t lfs_emubd_ssleep_t;
 
-// testbd config, this is required for testing
-struct lfs_testbd_config {
+// emubd config, this is required for testing
+struct lfs_emubd_config {
     // 8-bit erase value to use for simulating erases. -1 does not simulate
     // erases, which can speed up testing by avoiding the extra block-device
     // operations to store the erase value.
@@ -73,15 +77,15 @@ struct lfs_testbd_config {
     uint32_t erase_cycles;
 
     // The mode determining how bad-blocks fail
-    lfs_testbd_badblock_behavior_t badblock_behavior;
+    lfs_emubd_badblock_behavior_t badblock_behavior;
 
     // Number of write operations (erase/prog) before triggering a power-loss.
     // power_cycles=0 disables this. The exact behavior of power-loss is
     // controlled by a combination of powerloss_behavior and powerloss_cb.
-    lfs_testbd_powercycles_t power_cycles;
+    lfs_emubd_powercycles_t power_cycles;
 
     // The mode determining how power-loss affects disk
-    lfs_testbd_powerloss_behavior_t powerloss_behavior;
+    lfs_emubd_powerloss_behavior_t powerloss_behavior;
 
     // Function to call to emulate power-loss. The exact behavior of power-loss
     // is up to the runner to provide.
@@ -100,98 +104,119 @@ struct lfs_testbd_config {
 
     // Artificial delay in nanoseconds, there is no purpose for this other
     // than slowing down the simulation.
-    lfs_testbd_sleep_t read_sleep;
+    lfs_emubd_sleep_t read_sleep;
 
     // Artificial delay in nanoseconds, there is no purpose for this other
     // than slowing down the simulation.
-    lfs_testbd_sleep_t prog_sleep;
+    lfs_emubd_sleep_t prog_sleep;
 
     // Artificial delay in nanoseconds, there is no purpose for this other
     // than slowing down the simulation.
-    lfs_testbd_sleep_t erase_sleep;
+    lfs_emubd_sleep_t erase_sleep;
 };
 
 // A reference counted block
-typedef struct lfs_testbd_block {
+typedef struct lfs_emubd_block {
     uint32_t rc;
-    lfs_testbd_wear_t wear;
+    lfs_emubd_wear_t wear;
 
     uint8_t data[];
-} lfs_testbd_block_t;
+} lfs_emubd_block_t;
 
 // Disk mirror
-typedef struct lfs_testbd_disk {
+typedef struct lfs_emubd_disk {
     uint32_t rc;
     int fd;
     uint8_t *scratch;
-} lfs_testbd_disk_t;
+} lfs_emubd_disk_t;
 
-// testbd state
-typedef struct lfs_testbd {
+// emubd state
+typedef struct lfs_emubd {
     // array of copy-on-write blocks
-    lfs_testbd_block_t **blocks;
+    lfs_emubd_block_t **blocks;
 
     // some other test state
-    uint32_t power_cycles;
-    lfs_testbd_disk_t *disk;
+    lfs_emubd_io_t read;
+    lfs_emubd_io_t prog;
+    lfs_emubd_io_t erased;
+    lfs_emubd_powercycles_t power_cycles;
+    lfs_emubd_disk_t *disk;
 
-    const struct lfs_testbd_config *cfg;
-} lfs_testbd_t;
+    const struct lfs_emubd_config *cfg;
+} lfs_emubd_t;
 
 
 /// Block device API ///
 
-// Create a test block device using the geometry in lfs_config
+// Create an emulating block device using the geometry in lfs_config
 //
 // Note that filebd is used if a path is provided, if path is NULL
-// testbd will use rambd which can be much faster.
-int lfs_testbd_create(const struct lfs_config *cfg, const char *path);
-int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
-        const struct lfs_testbd_config *bdcfg);
+// emubd will use rambd which can be much faster.
+int lfs_emubd_create(const struct lfs_config *cfg, const char *path);
+int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path,
+        const struct lfs_emubd_config *bdcfg);
 
 // Clean up memory associated with block device
-int lfs_testbd_destroy(const struct lfs_config *cfg);
+int lfs_emubd_destroy(const struct lfs_config *cfg);
 
 // Read a block
-int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
+int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size);
 
 // Program a block
 //
 // The block must have previously been erased.
-int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
+int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size);
 
 // Erase a block
 //
 // A block must be erased before being programmed. The
 // state of an erased block is undefined.
-int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block);
+int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block);
 
 // Sync the block device
-int lfs_testbd_sync(const struct lfs_config *cfg);
+int lfs_emubd_sync(const struct lfs_config *cfg);
 
 
 /// Additional extended API for driving test features ///
 
+// Get total amount of bytes read
+lfs_emubd_sio_t lfs_emubd_getread(const struct lfs_config *cfg);
+
+// Get total amount of bytes programmed
+lfs_emubd_sio_t lfs_emubd_getprog(const struct lfs_config *cfg);
+
+// Get total amount of bytes erased
+lfs_emubd_sio_t lfs_emubd_geterased(const struct lfs_config *cfg);
+
+// Manually set amount of bytes read
+int lfs_emubd_setread(const struct lfs_config *cfg, lfs_emubd_io_t read);
+
+// Manually set amount of bytes programmed
+int lfs_emubd_setprog(const struct lfs_config *cfg, lfs_emubd_io_t prog);
+
+// Manually set amount of bytes erased
+int lfs_emubd_seterased(const struct lfs_config *cfg, lfs_emubd_io_t erased);
+
 // Get simulated wear on a given block
-lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg,
+lfs_emubd_swear_t lfs_emubd_getwear(const struct lfs_config *cfg,
         lfs_block_t block);
 
 // Manually set simulated wear on a given block
-int lfs_testbd_setwear(const struct lfs_config *cfg,
-        lfs_block_t block, lfs_testbd_wear_t wear);
+int lfs_emubd_setwear(const struct lfs_config *cfg,
+        lfs_block_t block, lfs_emubd_wear_t wear);
 
 // Get the remaining power-cycles
-lfs_testbd_spowercycles_t lfs_testbd_getpowercycles(
+lfs_emubd_spowercycles_t lfs_emubd_getpowercycles(
         const struct lfs_config *cfg);
 
 // Manually set the remaining power-cycles
-int lfs_testbd_setpowercycles(const struct lfs_config *cfg,
-        lfs_testbd_powercycles_t power_cycles);
+int lfs_emubd_setpowercycles(const struct lfs_config *cfg,
+        lfs_emubd_powercycles_t power_cycles);
 
 // Create a copy-on-write copy of the state of this block device
-int lfs_testbd_copy(const struct lfs_config *cfg, lfs_testbd_t *copy);
+int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy);
 
 
 #ifdef __cplusplus

benches/bench_dir.toml

@@ -0,0 +1,284 @@
+
+
+# deterministic prng
+code = '''
+static uint32_t xorshift32(uint32_t *state) {
+    uint32_t x = *state;
+    x ^= x << 13;
+    x ^= x >> 17;
+    x ^= x << 5;
+    *state = x;
+    return x;
+}
+'''
+
+[cases.bench_dir_open]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.N = 1024
+defines.FILE_SIZE = 8
+defines.CHUNK_SIZE = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    // first create the files
+    char name[256];
+    uint8_t buffer[CHUNK_SIZE];
+    for (lfs_size_t i = 0; i < N; i++) {
+        sprintf(name, "file%08x", i);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+        uint32_t file_prng = i;
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                buffer[k] = xorshift32(&file_prng);
+            }
+            lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+
+    // then read the files
+    BENCH_START();
+    uint32_t prng = 42;
+    for (lfs_size_t i = 0; i < N; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (N-1-i)
+            : xorshift32(&prng) % N;
+        sprintf(name, "file%08x", i_);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0;
+
+        uint32_t file_prng = i_;
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                assert(buffer[k] == xorshift32(&file_prng));
+            }
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_dir_creat]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.N = 1024
+defines.FILE_SIZE = 8
+defines.CHUNK_SIZE = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    BENCH_START();
+    uint32_t prng = 42;
+    char name[256];
+    uint8_t buffer[CHUNK_SIZE];
+    for (lfs_size_t i = 0; i < N; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (N-1-i)
+            : xorshift32(&prng) % N;
+        sprintf(name, "file%08x", i_);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
+
+        uint32_t file_prng = i_;
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                buffer[k] = xorshift32(&file_prng);
+            }
+            lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_dir_remove]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.N = 1024
+defines.FILE_SIZE = 8
+defines.CHUNK_SIZE = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    // first create the files
+    char name[256];
+    uint8_t buffer[CHUNK_SIZE];
+    for (lfs_size_t i = 0; i < N; i++) {
+        sprintf(name, "file%08x", i);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+        uint32_t file_prng = i;
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                buffer[k] = xorshift32(&file_prng);
+            }
+            lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+
+    // then remove the files
+    BENCH_START();
+    uint32_t prng = 42;
+    for (lfs_size_t i = 0; i < N; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (N-1-i)
+            : xorshift32(&prng) % N;
+        sprintf(name, "file%08x", i_);
+        int err = lfs_remove(&lfs, name);
+        assert(!err || err == LFS_ERR_NOENT);
+    }
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_dir_read]
+defines.N = 1024
+defines.FILE_SIZE = 8
+defines.CHUNK_SIZE = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    // first create the files
+    char name[256];
+    uint8_t buffer[CHUNK_SIZE];
+    for (lfs_size_t i = 0; i < N; i++) {
+        sprintf(name, "file%08x", i);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+        uint32_t file_prng = i;
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                buffer[k] = xorshift32(&file_prng);
+            }
+            lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+
+    // then read the directory
+    BENCH_START();
+    lfs_dir_t dir;
+    lfs_dir_open(&lfs, &dir, "/") => 0;
+    struct lfs_info info;
+    lfs_dir_read(&lfs, &dir, &info) => 1;
+    assert(info.type == LFS_TYPE_DIR);
+    assert(strcmp(info.name, ".") == 0);
+    lfs_dir_read(&lfs, &dir, &info) => 1;
+    assert(info.type == LFS_TYPE_DIR);
+    assert(strcmp(info.name, "..") == 0);
+    for (int i = 0; i < N; i++) {
+        sprintf(name, "file%08x", i);
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        assert(info.type == LFS_TYPE_REG);
+        assert(strcmp(info.name, name) == 0);
+    }
+    lfs_dir_read(&lfs, &dir, &info) => 0;
+    lfs_dir_close(&lfs, &dir) => 0;
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_dir_mkdir]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.N = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    BENCH_START();
+    uint32_t prng = 42;
+    char name[256];
+    for (lfs_size_t i = 0; i < N; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (N-1-i)
+            : xorshift32(&prng) % N;
+        printf("hm %d\n", i);
+        sprintf(name, "dir%08x", i_);
+        int err = lfs_mkdir(&lfs, name);
+        assert(!err || err == LFS_ERR_EXIST);
+    }
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_dir_rmdir]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.N = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+
+    // first create the dirs
+    char name[256];
+    for (lfs_size_t i = 0; i < N; i++) {
+        sprintf(name, "dir%08x", i);
+        lfs_mkdir(&lfs, name) => 0;
+    }
+
+    // then remove the dirs
+    BENCH_START();
+    uint32_t prng = 42;
+    for (lfs_size_t i = 0; i < N; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (N-1-i)
+            : xorshift32(&prng) % N;
+        sprintf(name, "dir%08x", i_);
+        int err = lfs_remove(&lfs, name);
+        assert(!err || err == LFS_ERR_NOENT);
+    }
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+

benches/bench_file.toml

@@ -0,0 +1,109 @@
+
+
+# deterministic prng
+code = '''
+static uint32_t xorshift32(uint32_t *state) {
+    uint32_t x = *state;
+    x ^= x << 13;
+    x ^= x >> 17;
+    x ^= x << 5;
+    *state = x;
+    return x;
+}
+'''
+
+[cases.bench_file_read]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.SIZE = '128*1024'
+defines.CHUNK_SIZE = 64
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+    lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE;
+
+    // first write the file
+    lfs_file_t file;
+    uint8_t buffer[CHUNK_SIZE];
+    lfs_file_open(&lfs, &file, "file",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    for (lfs_size_t i = 0; i < chunks; i++) {
+        uint32_t chunk_prng = i;
+        for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
+            buffer[j] = xorshift32(&chunk_prng);
+        }
+
+        lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+    }
+    lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+    lfs_file_close(&lfs, &file) => 0;
+
+    // then read the file
+    BENCH_START();
+    lfs_file_open(&lfs, &file, "file", LFS_O_RDONLY) => 0;
+
+    uint32_t prng = 42;
+    for (lfs_size_t i = 0; i < chunks; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (chunks-1-i)
+            : xorshift32(&prng) % chunks;
+        lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)
+                => i_*CHUNK_SIZE;
+        lfs_file_read(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+
+        uint32_t chunk_prng = i_;
+        for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
+            assert(buffer[j] == xorshift32(&chunk_prng));
+        }
+    }
+
+    lfs_file_close(&lfs, &file) => 0;
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_file_write]
+# 0 = in-order
+# 1 = reversed-order
+# 2 = random-order
+defines.ORDER = [0, 1, 2]
+defines.SIZE = '128*1024'
+defines.CHUNK_SIZE = 64
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+    lfs_mount(&lfs, cfg) => 0;
+    lfs_size_t chunks = (SIZE+CHUNK_SIZE-1)/CHUNK_SIZE;
+
+    BENCH_START();
+    lfs_file_t file;
+    lfs_file_open(&lfs, &file, "file",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+    uint8_t buffer[CHUNK_SIZE];
+    uint32_t prng = 42;
+    for (lfs_size_t i = 0; i < chunks; i++) {
+        lfs_off_t i_
+            = (ORDER == 0) ? i
+            : (ORDER == 1) ? (chunks-1-i)
+            : xorshift32(&prng) % chunks;
+        uint32_t chunk_prng = i_;
+        for (lfs_size_t j = 0; j < CHUNK_SIZE; j++) {
+            buffer[j] = xorshift32(&chunk_prng);
+        }
+
+        lfs_file_seek(&lfs, &file, i_*CHUNK_SIZE, LFS_SEEK_SET)
+                => i_*CHUNK_SIZE;
+        lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+    }
+
+    lfs_file_close(&lfs, &file) => 0;
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''

benches/bench_superblock.toml

@@ -0,0 +1,56 @@
+[cases.bench_superblocks_found]
+# support benchmarking with files
+defines.N = [0, 1024]
+defines.FILE_SIZE = 8
+defines.CHUNK_SIZE = 8
+code = '''
+    lfs_t lfs;
+    lfs_format(&lfs, cfg) => 0;
+
+    // create files?
+    lfs_mount(&lfs, cfg) => 0;
+    char name[256];
+    uint8_t buffer[CHUNK_SIZE];
+    for (lfs_size_t i = 0; i < N; i++) {
+        sprintf(name, "file%08x", i);
+        lfs_file_t file;
+        lfs_file_open(&lfs, &file, name,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+
+        for (lfs_size_t j = 0; j < FILE_SIZE; j += CHUNK_SIZE) {
+            for (lfs_size_t k = 0; k < CHUNK_SIZE; k++) {
+                buffer[k] = i+j+k;
+            }
+            lfs_file_write(&lfs, &file, buffer, CHUNK_SIZE) => CHUNK_SIZE;
+        }
+
+        lfs_file_close(&lfs, &file) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+
+    BENCH_START();
+    lfs_mount(&lfs, cfg) => 0;
+    BENCH_STOP();
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[cases.bench_superblocks_missing]
+code = '''
+    lfs_t lfs;
+
+    BENCH_START();
+    int err = lfs_mount(&lfs, cfg);
+    assert(err != 0);
+    BENCH_STOP();
+'''
+
+[cases.bench_superblocks_format]
+code = '''
+    lfs_t lfs;
+
+    BENCH_START();
+    lfs_format(&lfs, cfg) => 0;
+    BENCH_STOP();
+'''
+

runners/bench_runner.c

@@ -0,0 +1,1795 @@
+
+#ifndef _POSIX_C_SOURCE
+#define _POSIX_C_SOURCE 199309L
+#endif
+
+#include "runners/bench_runner.h"
+#include "bd/lfs_emubd.h"
+
+#include <getopt.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <setjmp.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <unistd.h>
+
+
+// some helpers
+
+// append to an array with amortized doubling
+void *mappend(void **p,
+        size_t size,
+        size_t *count,
+        size_t *capacity) {
+    uint8_t *p_ = *p;
+    size_t count_ = *count;
+    size_t capacity_ = *capacity;
+
+    count_ += 1;
+    if (count_ > capacity_) {
+        capacity_ = (2*capacity_ < 4) ? 4 : 2*capacity_;
+
+        p_ = realloc(p_, capacity_*size);
+        if (!p_) {
+            return NULL;
+        }
+    }
+
+    *p = p_;
+    *count = count_;
+    *capacity = capacity_;
+    return &p_[(count_-1)*size];
+}
+
+// a quick self-terminating text-safe varint scheme
+static void leb16_print(uintmax_t x) {
+    while (true) {
+        char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
+        printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
+        if (x <= 0xf) {
+            break;
+        }
+        x >>= 4;
+    }
+}
+
+static uintmax_t leb16_parse(const char *s, char **tail) {
+    uintmax_t x = 0;
+    size_t i = 0;
+    while (true) {
+        uintmax_t nibble = s[i];
+        if (nibble >= '0' && nibble <= '9') {
+            nibble = nibble - '0';
+        } else if (nibble >= 'a' && nibble <= 'v') {
+            nibble = nibble - 'a' + 10;
+        } else {
+            // invalid?
+            if (tail) {
+                *tail = (char*)s;
+            }
+            return 0;
+        }
+
+        x |= (nibble & 0xf) << (4*i);
+        i += 1;
+        if (!(nibble & 0x10)) {
+            break;
+        }
+    }
+
+    if (tail) {
+        *tail = (char*)s + i;
+    }
+    return x;
+}
+
+
+
+// bench_runner types
+
+typedef struct bench_geometry {
+    char short_name;
+    const char *long_name;
+
+    bench_define_t defines[BENCH_GEOMETRY_DEFINE_COUNT];
+} bench_geometry_t;
+
+typedef struct bench_id {
+    const char *name;
+    const bench_define_t *defines;
+    size_t define_count;
+} bench_id_t;
+
+
+// bench suites are linked into a custom ld section
+extern struct bench_suite __start__bench_suites;
+extern struct bench_suite __stop__bench_suites;
+
+const struct bench_suite *bench_suites = &__start__bench_suites;
+#define BENCH_SUITE_COUNT \
+    ((size_t)(&__stop__bench_suites - &__start__bench_suites))
+
+
+// bench define management
+typedef struct bench_define_map {
+    const bench_define_t *defines;
+    size_t count;
+} bench_define_map_t;
+
+typedef struct bench_define_names {
+    const char *const *names;
+    size_t count;
+} bench_define_names_t;
+
+intmax_t bench_define_lit(void *data) {
+    return (intmax_t)data;
+}
+
+#define BENCH_CONST(x) {bench_define_lit, (void*)(uintptr_t)(x)}
+#define BENCH_LIT(x) ((bench_define_t)BENCH_CONST(x))
+
+
+#define BENCH_DEF(k, v) \
+    intmax_t bench_define_##k(void *data) { \
+        (void)data; \
+        return v; \
+    }
+
+    BENCH_IMPLICIT_DEFINES
+#undef BENCH_DEF
+
+#define BENCH_DEFINE_MAP_EXPLICIT    0
+#define BENCH_DEFINE_MAP_OVERRIDE    1
+#define BENCH_DEFINE_MAP_PERMUTATION 2
+#define BENCH_DEFINE_MAP_GEOMETRY    3
+#define BENCH_DEFINE_MAP_IMPLICIT    4
+#define BENCH_DEFINE_MAP_COUNT       5
+
+bench_define_map_t bench_define_maps[BENCH_DEFINE_MAP_COUNT] = {
+    [BENCH_DEFINE_MAP_IMPLICIT] = {
+        (const bench_define_t[BENCH_IMPLICIT_DEFINE_COUNT]) {
+            #define BENCH_DEF(k, v) \
+                [k##_i] = {bench_define_##k, NULL},
+
+                BENCH_IMPLICIT_DEFINES
+            #undef BENCH_DEF
+        },
+        BENCH_IMPLICIT_DEFINE_COUNT,
+    },
+};
+
+#define BENCH_DEFINE_NAMES_SUITE    0
+#define BENCH_DEFINE_NAMES_IMPLICIT 1
+#define BENCH_DEFINE_NAMES_COUNT    2
+
+bench_define_names_t bench_define_names[BENCH_DEFINE_NAMES_COUNT] = {
+    [BENCH_DEFINE_NAMES_IMPLICIT] = {
+        (const char *const[BENCH_IMPLICIT_DEFINE_COUNT]){
+            #define BENCH_DEF(k, v) \
+                [k##_i] = #k,
+
+                BENCH_IMPLICIT_DEFINES
+            #undef BENCH_DEF
+        },
+        BENCH_IMPLICIT_DEFINE_COUNT,
+    },
+};
+
+intmax_t *bench_define_cache;
+size_t bench_define_cache_count;
+unsigned *bench_define_cache_mask;
+
+const char *bench_define_name(size_t define) {
+    // lookup in our bench names
+    for (size_t i = 0; i < BENCH_DEFINE_NAMES_COUNT; i++) {
+        if (define < bench_define_names[i].count
+                && bench_define_names[i].names
+                && bench_define_names[i].names[define]) {
+            return bench_define_names[i].names[define];
+        }
+    }
+
+    return NULL;
+}
+
+bool bench_define_ispermutation(size_t define) {
+    // is this define specific to the permutation?
+    for (size_t i = 0; i < BENCH_DEFINE_MAP_IMPLICIT; i++) {
+        if (define < bench_define_maps[i].count
+                && bench_define_maps[i].defines[define].cb) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+intmax_t bench_define(size_t define) {
+    // is the define in our cache?
+    if (define < bench_define_cache_count
+            && (bench_define_cache_mask[define/(8*sizeof(unsigned))]
+                & (1 << (define%(8*sizeof(unsigned)))))) {
+        return bench_define_cache[define];
+    }
+
+    // lookup in our bench defines
+    for (size_t i = 0; i < BENCH_DEFINE_MAP_COUNT; i++) {
+        if (define < bench_define_maps[i].count
+                && bench_define_maps[i].defines[define].cb) {
+            intmax_t v = bench_define_maps[i].defines[define].cb(
+                    bench_define_maps[i].defines[define].data);
+
+            // insert into cache!
+            bench_define_cache[define] = v;
+            bench_define_cache_mask[define / (8*sizeof(unsigned))]
+                    |= 1 << (define%(8*sizeof(unsigned)));
+
+            return v;
+        }
+    }
+
+    return 0;
+
+    // not found?
+    const char *name = bench_define_name(define);
+    fprintf(stderr, "error: undefined define %s (%zd)\n",
+            name ? name : "(unknown)",
+            define);
+    assert(false);
+    exit(-1);
+}
+
+void bench_define_flush(void) {
+    // clear cache between permutations
+    memset(bench_define_cache_mask, 0,
+            sizeof(unsigned)*(
+                (bench_define_cache_count+(8*sizeof(unsigned))-1)
+                / (8*sizeof(unsigned))));
+}
+
+// geometry updates
+const bench_geometry_t *bench_geometry = NULL;
+
+void bench_define_geometry(const bench_geometry_t *geometry) {
+    bench_define_maps[BENCH_DEFINE_MAP_GEOMETRY] = (bench_define_map_t){
+            geometry->defines, BENCH_GEOMETRY_DEFINE_COUNT};
+}
+
+// override updates
+typedef struct bench_override {
+    const char *name;
+    const intmax_t *defines;
+    size_t permutations;
+} bench_override_t;
+
+const bench_override_t *bench_overrides = NULL;
+size_t bench_override_count = 0;
+
+bench_define_t *bench_override_defines = NULL;
+size_t bench_override_define_count = 0;
+size_t bench_override_define_permutations = 1;
+size_t bench_override_define_capacity = 0;
+
+// suite/perm updates
+void bench_define_suite(const struct bench_suite *suite) {
+    bench_define_names[BENCH_DEFINE_NAMES_SUITE] = (bench_define_names_t){
+            suite->define_names, suite->define_count};
+
+    // make sure our cache is large enough
+    if (lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT)
+            > bench_define_cache_count) {
+        // align to power of two to avoid any superlinear growth
+        size_t ncount = 1 << lfs_npw2(
+                lfs_max(suite->define_count, BENCH_IMPLICIT_DEFINE_COUNT));
+        bench_define_cache = realloc(bench_define_cache, ncount*sizeof(intmax_t));
+        bench_define_cache_mask = realloc(bench_define_cache_mask,
+                sizeof(unsigned)*(
+                    (ncount+(8*sizeof(unsigned))-1)
+                    / (8*sizeof(unsigned))));
+        bench_define_cache_count = ncount;
+    }
+
+    // map any overrides
+    if (bench_override_count > 0) {
+        // first figure out the total size of override permutations
+        size_t count = 0;
+        size_t permutations = 1;
+        for (size_t i = 0; i < bench_override_count; i++) {
+            for (size_t d = 0;
+                    d < lfs_max(
+                        suite->define_count,
+                        BENCH_IMPLICIT_DEFINE_COUNT);
+                    d++) {
+                // define name match?
+                const char *name = bench_define_name(d);
+                if (name && strcmp(name, bench_overrides[i].name) == 0) {
+                    count = lfs_max(count, d+1);
+                    permutations *= bench_overrides[i].permutations;
+                    break;
+                }
+            }
+        }
+        bench_override_define_count = count;
+        bench_override_define_permutations = permutations;
+
+        // make sure our override arrays are big enough
+        if (count * permutations > bench_override_define_capacity) {
+            // align to power of two to avoid any superlinear growth
+            size_t ncapacity = 1 << lfs_npw2(count * permutations);
+            bench_override_defines = realloc(
+                    bench_override_defines,
+                    sizeof(bench_define_t)*ncapacity);
+            bench_override_define_capacity = ncapacity;
+        }
+
+        // zero unoverridden defines
+        memset(bench_override_defines, 0,
+                sizeof(bench_define_t) * count * permutations);
+
+        // compute permutations
+        size_t p = 1;
+        for (size_t i = 0; i < bench_override_count; i++) {
+            for (size_t d = 0;
+                    d < lfs_max(
+                        suite->define_count,
+                        BENCH_IMPLICIT_DEFINE_COUNT);
+                    d++) {
+                // define name match?
+                const char *name = bench_define_name(d);
+                if (name && strcmp(name, bench_overrides[i].name) == 0) {
+                    // scatter the define permutations based on already
+                    // seen permutations
+                    for (size_t j = 0; j < permutations; j++) {
+                        bench_override_defines[j*count + d] = BENCH_LIT(
+                                bench_overrides[i].defines[(j/p)
+                                    % bench_overrides[i].permutations]);
+                    }
+
+                    // keep track of how many permutations we've seen so far
+                    p *= bench_overrides[i].permutations;
+                    break;
+                }
+            }
+        }
+    }
+}
+
+void bench_define_perm(
+        const struct bench_suite *suite,
+        const struct bench_case *case_,
+        size_t perm) {
+    if (case_->defines) {
+        bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){
+                case_->defines + perm*suite->define_count,
+                suite->define_count};
+    } else {
+        bench_define_maps[BENCH_DEFINE_MAP_PERMUTATION] = (bench_define_map_t){
+                NULL, 0};
+    }
+}
+
+void bench_define_override(size_t perm) {
+    bench_define_maps[BENCH_DEFINE_MAP_OVERRIDE] = (bench_define_map_t){
+            bench_override_defines + perm*bench_override_define_count,
+            bench_override_define_count};
+}
+
+void bench_define_explicit(
+        const bench_define_t *defines,
+        size_t define_count) {
+    bench_define_maps[BENCH_DEFINE_MAP_EXPLICIT] = (bench_define_map_t){
+            defines, define_count};
+}
+
+void bench_define_cleanup(void) {
+    // bench define management can allocate a few things
+    free(bench_define_cache);
+    free(bench_define_cache_mask);
+    free(bench_override_defines);
+}
+
+
+
+// bench state
+extern const bench_geometry_t *bench_geometries;
+extern size_t bench_geometry_count;
+
+const bench_id_t *bench_ids = (const bench_id_t[]) {
+    {NULL, NULL, 0},
+};
+size_t bench_id_count = 1;
+
+size_t bench_step_start = 0;
+size_t bench_step_stop = -1;
+size_t bench_step_step = 1;
+
+const char *bench_disk_path = NULL;
+const char *bench_trace_path = NULL;
+FILE *bench_trace_file = NULL;
+uint32_t bench_trace_cycles = 0;
+lfs_emubd_sleep_t bench_read_sleep = 0.0;
+lfs_emubd_sleep_t bench_prog_sleep = 0.0;
+lfs_emubd_sleep_t bench_erase_sleep = 0.0;
+
+
+// trace printing
+void bench_trace(const char *fmt, ...) {
+    if (bench_trace_path) {
+        if (!bench_trace_file) {
+            // Tracing output is heavy and trying to open every trace
+            // call is slow, so we only try to open the trace file every
+            // so often. Note this doesn't affect successfully opened files
+            if (bench_trace_cycles % 128 != 0) {
+                bench_trace_cycles += 1;
+                return;
+            }
+            bench_trace_cycles += 1;
+
+            int fd;
+            if (strcmp(bench_trace_path, "-") == 0) {
+                fd = dup(1);
+                if (fd < 0) {
+                    return;
+                }
+            } else {
+                fd = open(
+                        bench_trace_path,
+                        O_WRONLY | O_CREAT | O_APPEND | O_NONBLOCK,
+                        0666);
+                if (fd < 0) {
+                    return;
+                }
+                int err = fcntl(fd, F_SETFL, O_WRONLY | O_CREAT | O_APPEND);
+                assert(!err);
+            }
+
+            FILE *f = fdopen(fd, "a");
+            assert(f);
+            int err = setvbuf(f, NULL, _IOLBF, BUFSIZ);
+            assert(!err);
+            bench_trace_file = f;
+        }
+
+        va_list va;
+        va_start(va, fmt);
+        int res = vfprintf(bench_trace_file, fmt, va);
+        if (res < 0) {
+            fclose(bench_trace_file);
+            bench_trace_file = NULL;
+        }
+        va_end(va);
+    }
+}
+
+
+// bench recording state
+static struct lfs_config *bench_cfg = NULL;
+static lfs_emubd_io_t bench_last_read = 0;
+static lfs_emubd_io_t bench_last_prog = 0;
+static lfs_emubd_io_t bench_last_erased = 0;
+lfs_emubd_io_t bench_read = 0;
+lfs_emubd_io_t bench_prog = 0;
+lfs_emubd_io_t bench_erased = 0;
+
+void bench_reset(void) {
+    bench_read = 0;
+    bench_prog = 0;
+    bench_erased = 0;
+    bench_last_read = 0;
+    bench_last_prog = 0;
+    bench_last_erased = 0;
+}
+
+void bench_start(void) {
+    assert(bench_cfg);
+    lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg);
+    assert(read >= 0);
+    lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg);
+    assert(prog >= 0);
+    lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg);
+    assert(erased >= 0);
+
+    bench_last_read = read;
+    bench_last_prog = prog;
+    bench_last_erased = erased;
+}
+
+void bench_stop(void) {
+    assert(bench_cfg);
+    lfs_emubd_sio_t read = lfs_emubd_getread(bench_cfg);
+    assert(read >= 0);
+    lfs_emubd_sio_t prog = lfs_emubd_getprog(bench_cfg);
+    assert(prog >= 0);
+    lfs_emubd_sio_t erased = lfs_emubd_geterased(bench_cfg);
+    assert(erased >= 0);
+
+    bench_read += read - bench_last_read;
+    bench_prog += prog - bench_last_prog;
+    bench_erased += erased - bench_last_erased;
+}
+
+
+// encode our permutation into a reusable id
+static void perm_printid(
+        const struct bench_suite *suite,
+        const struct bench_case *case_) {
+    (void)suite;
+    // case[:permutation]
+    printf("%s:", case_->name);
+    for (size_t d = 0;
+            d < lfs_max(
+                suite->define_count,
+                BENCH_IMPLICIT_DEFINE_COUNT);
+            d++) {
+        if (bench_define_ispermutation(d)) {
+            leb16_print(d);
+            leb16_print(BENCH_DEFINE(d));
+        }
+    }
+}
+
+// iterate through permutations in a bench case
+static void case_forperm(
+        const struct bench_suite *suite,
+        const struct bench_case *case_,
+        const bench_define_t *defines,
+        size_t define_count,
+        void (*cb)(
+            void *data,
+            const struct bench_suite *suite,
+            const struct bench_case *case_),
+        void *data) {
+    if (defines) {
+        bench_define_explicit(defines, define_count);
+        bench_define_flush();
+
+        cb(data, suite, case_);
+    } else {
+        for (size_t k = 0; k < case_->permutations; k++) {
+            // define permutation
+            bench_define_perm(suite, case_, k);
+
+            for (size_t v = 0; v < bench_override_define_permutations; v++) {
+                // define override permutation
+                bench_define_override(v);
+
+                for (size_t g = 0; g < bench_geometry_count; g++) {
+                    // define geometry
+                    bench_define_geometry(&bench_geometries[g]);
+                    bench_define_flush();
+
+                    cb(data, suite, case_);
+                }
+            }
+        }
+    }
+}
+
+
+// how many permutations are there actually in a bench case
+struct perm_count_state {
+    size_t total;
+    size_t filtered;
+};
+
+void perm_count(
+        void *data,
+        const struct bench_suite *suite,
+        const struct bench_case *case_) {
+    struct perm_count_state *state = data;
+    (void)suite;
+    (void)case_;
+
+    state->total += 1;
+
+    if (case_->filter && !case_->filter()) {
+        return;
+    }
+
+    state->filtered += 1;
+}
+
+
+// operations we can do
+static void summary(void) {
+    printf("%-36s %7s %7s %7s %11s\n",
+            "", "flags", "suites", "cases", "perms");
+    size_t suites = 0;
+    size_t cases = 0;
+    bench_flags_t flags = 0;
+    struct perm_count_state perms = {0, 0};
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                cases += 1;
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_count,
+                        &perms);
+            }
+
+            suites += 1;
+            flags |= bench_suites[i].flags;
+        }
+    }
+
+    char perm_buf[64];
+    sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
+    char flag_buf[64];
+    sprintf(flag_buf, "%s%s",
+            (flags & BENCH_REENTRANT) ? "r" : "",
+            (!flags) ? "-" : "");
+    printf("%-36s %7s %7zu %7zu %11s\n",
+            "TOTAL",
+            flag_buf,
+            suites,
+            cases,
+            perm_buf);
+}
+
+static void list_suites(void) {
+    printf("%-36s %7s %7s %11s\n", "suite", "flags", "cases", "perms");
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            size_t cases = 0;
+            struct perm_count_state perms = {0, 0};
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                cases += 1;
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_count,
+                        &perms);
+            }
+
+            // no benches found?
+            if (!cases) {
+                continue;
+            }
+
+            char perm_buf[64];
+            sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
+            char flag_buf[64];
+            sprintf(flag_buf, "%s%s",
+                    (bench_suites[i].flags & BENCH_REENTRANT) ? "r" : "",
+                    (!bench_suites[i].flags) ? "-" : "");
+            printf("%-36s %7s %7zu %11s\n",
+                    bench_suites[i].name,
+                    flag_buf,
+                    cases,
+                    perm_buf);
+        }
+    }
+}
+
+static void list_cases(void) {
+    printf("%-36s %7s %11s\n", "case", "flags", "perms");
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                struct perm_count_state perms = {0, 0};
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_count,
+                        &perms);
+
+                char perm_buf[64];
+                sprintf(perm_buf, "%zu/%zu", perms.filtered, perms.total);
+                char flag_buf[64];
+                sprintf(flag_buf, "%s%s",
+                        (bench_suites[i].cases[j].flags & BENCH_REENTRANT)
+                            ? "r" : "",
+                        (!bench_suites[i].cases[j].flags)
+                            ? "-" : "");
+                printf("%-36s %7s %11s\n",
+                        bench_suites[i].cases[j].name,
+                        flag_buf,
+                        perm_buf);
+            }
+        }
+    }
+}
+
+static void list_suite_paths(void) {
+    printf("%-36s %s\n", "suite", "path");
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            size_t cases = 0;
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+            }
+
+            // no benches found?
+            if (!cases) {
+                continue;
+            }
+
+            printf("%-36s %s\n",
+                    bench_suites[i].name,
+                    bench_suites[i].path);
+        }
+    }
+}
+
+static void list_case_paths(void) {
+    printf("%-36s %s\n", "case", "path");
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                printf("%-36s %s\n",
+                        bench_suites[i].cases[j].name,
+                        bench_suites[i].cases[j].path);
+            }
+        }
+    }
+}
+
+struct list_defines_define {
+    const char *name;
+    intmax_t *values;
+    size_t value_count;
+    size_t value_capacity;
+};
+
+struct list_defines_defines {
+    struct list_defines_define *defines;
+    size_t define_count;
+    size_t define_capacity;
+};
+
+static void list_defines_add(
+        struct list_defines_defines *defines,
+        size_t d) {
+    const char *name = bench_define_name(d);
+    intmax_t value = BENCH_DEFINE(d);
+
+    // define already in defines?
+    for (size_t i = 0; i < defines->define_count; i++) {
+        if (strcmp(defines->defines[i].name, name) == 0) {
+            // value already in values?
+            for (size_t j = 0; j < defines->defines[i].value_count; j++) {
+                if (defines->defines[i].values[j] == value) {
+                    return;
+                }
+            }
+
+            *(intmax_t*)mappend(
+                (void**)&defines->defines[i].values,
+                sizeof(intmax_t),
+                &defines->defines[i].value_count,
+                &defines->defines[i].value_capacity) = value;
+
+            return;
+        }
+    }
+
+    // new define?
+    struct list_defines_define *define = mappend(
+            (void**)&defines->defines,
+            sizeof(struct list_defines_define),
+            &defines->define_count,
+            &defines->define_capacity);
+    define->name = name;
+    define->values = malloc(sizeof(intmax_t));
+    define->values[0] = value;
+    define->value_count = 1;
+    define->value_capacity = 1;
+}
+
+void perm_list_defines(
+        void *data,
+        const struct bench_suite *suite,
+        const struct bench_case *case_) {
+    struct list_defines_defines *defines = data;
+    (void)suite;
+    (void)case_;
+
+    // collect defines
+    for (size_t d = 0;
+            d < lfs_max(suite->define_count,
+                BENCH_IMPLICIT_DEFINE_COUNT);
+            d++) {
+        if (d < BENCH_IMPLICIT_DEFINE_COUNT
+                || bench_define_ispermutation(d)) {
+            list_defines_add(defines, d);
+        }
+    }
+}
+
+void perm_list_permutation_defines(
+        void *data,
+        const struct bench_suite *suite,
+        const struct bench_case *case_) {
+    struct list_defines_defines *defines = data;
+    (void)suite;
+    (void)case_;
+
+    // collect permutation_defines
+    for (size_t d = 0;
+            d < lfs_max(suite->define_count,
+                BENCH_IMPLICIT_DEFINE_COUNT);
+            d++) {
+        if (bench_define_ispermutation(d)) {
+            list_defines_add(defines, d);
+        }
+    }
+}
+
+extern const bench_geometry_t builtin_geometries[];
+
+static void list_defines(void) {
+    struct list_defines_defines defines = {NULL, 0, 0};
+
+    // add defines
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_list_defines,
+                        &defines);
+            }
+        }
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        printf("%s=", defines.defines[i].name);
+        for (size_t j = 0; j < defines.defines[i].value_count; j++) {
+            printf("%jd", defines.defines[i].values[j]);
+            if (j != defines.defines[i].value_count-1) {
+                printf(",");
+            }
+        }
+        printf("\n");
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        free(defines.defines[i].values);
+    }
+    free(defines.defines);
+}
+
+static void list_permutation_defines(void) {
+    struct list_defines_defines defines = {NULL, 0, 0};
+
+    // add permutation defines
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_list_permutation_defines,
+                        &defines);
+            }
+        }
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        printf("%s=", defines.defines[i].name);
+        for (size_t j = 0; j < defines.defines[i].value_count; j++) {
+            printf("%jd", defines.defines[i].values[j]);
+            if (j != defines.defines[i].value_count-1) {
+                printf(",");
+            }
+        }
+        printf("\n");
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        free(defines.defines[i].values);
+    }
+    free(defines.defines);
+}
+
+static void list_implicit_defines(void) {
+    struct list_defines_defines defines = {NULL, 0, 0};
+
+    // yes we do need to define a suite, this does a bit of bookeeping
+    // such as setting up the define cache
+    bench_define_suite(&(const struct bench_suite){0});
+
+    // make sure to include builtin geometries here
+    extern const bench_geometry_t builtin_geometries[];
+    for (size_t g = 0; builtin_geometries[g].long_name; g++) {
+        bench_define_geometry(&builtin_geometries[g]);
+        bench_define_flush();
+
+        // add implicit defines
+        for (size_t d = 0; d < BENCH_IMPLICIT_DEFINE_COUNT; d++) {
+            list_defines_add(&defines, d);
+        }
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        printf("%s=", defines.defines[i].name);
+        for (size_t j = 0; j < defines.defines[i].value_count; j++) {
+            printf("%jd", defines.defines[i].values[j]);
+            if (j != defines.defines[i].value_count-1) {
+                printf(",");
+            }
+        }
+        printf("\n");
+    }
+
+    for (size_t i = 0; i < defines.define_count; i++) {
+        free(defines.defines[i].values);
+    }
+    free(defines.defines);
+}
+
+
+
+// geometries to bench
+
+const bench_geometry_t builtin_geometries[] = {
+    {'d', "default", {{NULL}, BENCH_CONST(16),   BENCH_CONST(512),   {NULL}}},
+    {'e', "eeprom",  {{NULL}, BENCH_CONST(1),    BENCH_CONST(512),   {NULL}}},
+    {'E', "emmc",    {{NULL}, {NULL},            BENCH_CONST(512),   {NULL}}},
+    {'n', "nor",     {{NULL}, BENCH_CONST(1),    BENCH_CONST(4096),  {NULL}}},
+    {'N', "nand",    {{NULL}, BENCH_CONST(4096), BENCH_CONST(32768), {NULL}}},
+    {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}},
+};
+
+const bench_geometry_t *bench_geometries = builtin_geometries;
+size_t bench_geometry_count = 5;
+
+static void list_geometries(void) {
+    // yes we do need to define a suite, this does a bit of bookeeping
+    // such as setting up the define cache
+    bench_define_suite(&(const struct bench_suite){0});
+
+    printf("%-24s %7s %7s %7s %7s %11s\n",
+            "geometry", "read", "prog", "erase", "count", "size");
+    for (size_t g = 0; builtin_geometries[g].long_name; g++) {
+        bench_define_geometry(&builtin_geometries[g]);
+        bench_define_flush();
+        printf("%c,%-22s %7ju %7ju %7ju %7ju %11ju\n",
+                builtin_geometries[g].short_name,
+                builtin_geometries[g].long_name,
+                READ_SIZE,
+                PROG_SIZE,
+                BLOCK_SIZE,
+                BLOCK_COUNT,
+                BLOCK_SIZE*BLOCK_COUNT);
+    }
+}
+
+
+
+// global bench step count
+size_t bench_step = 0;
+
+void perm_run(
+        void *data,
+        const struct bench_suite *suite,
+        const struct bench_case *case_) {
+    (void)data;
+
+    // skip this step?
+    if (!(bench_step >= bench_step_start
+            && bench_step < bench_step_stop
+            && (bench_step-bench_step_start) % bench_step_step == 0)) {
+        bench_step += 1;
+        return;
+    }
+    bench_step += 1;
+
+    // filter?
+    if (case_->filter && !case_->filter()) {
+        printf("skipped ");
+        perm_printid(suite, case_);
+        printf("\n");
+        return;
+    }
+
+    // create block device and configuration
+    lfs_emubd_t bd;
+
+    struct lfs_config cfg = {
+        .context            = &bd,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
+        .read_size          = READ_SIZE,
+        .prog_size          = PROG_SIZE,
+        .block_size         = BLOCK_SIZE,
+        .block_count        = BLOCK_COUNT,
+        .block_cycles       = BLOCK_CYCLES,
+        .cache_size         = CACHE_SIZE,
+        .lookahead_size     = LOOKAHEAD_SIZE,
+    };
+
+    struct lfs_emubd_config bdcfg = {
+        .erase_value        = ERASE_VALUE,
+        .erase_cycles       = ERASE_CYCLES,
+        .badblock_behavior  = BADBLOCK_BEHAVIOR,
+        .disk_path          = bench_disk_path,
+        .read_sleep         = bench_read_sleep,
+        .prog_sleep         = bench_prog_sleep,
+        .erase_sleep        = bench_erase_sleep,
+    };
+
+    int err = lfs_emubd_createcfg(&cfg, bench_disk_path, &bdcfg);
+    if (err) {
+        fprintf(stderr, "error: could not create block device: %d\n", err);
+        exit(-1);
+    }
+
+    // run the bench
+    bench_cfg = &cfg;
+    bench_reset();
+    printf("running ");
+    perm_printid(suite, case_);
+    printf("\n");
+
+    case_->run(&cfg);
+
+    printf("finished ");
+    perm_printid(suite, case_);
+    printf(" %"PRIu64" %"PRIu64" %"PRIu64,
+        bench_read,
+        bench_prog,
+        bench_erased);
+    printf("\n");
+
+    // cleanup
+    err = lfs_emubd_destroy(&cfg);
+    if (err) {
+        fprintf(stderr, "error: could not destroy block device: %d\n", err);
+        exit(-1);
+    }
+}
+
+static void run(void) {
+    // ignore disconnected pipes
+    signal(SIGPIPE, SIG_IGN);
+
+    for (size_t t = 0; t < bench_id_count; t++) {
+        for (size_t i = 0; i < BENCH_SUITE_COUNT; i++) {
+            bench_define_suite(&bench_suites[i]);
+
+            for (size_t j = 0; j < bench_suites[i].case_count; j++) {
+                // does neither suite nor case name match?
+                if (bench_ids[t].name && !(
+                        strcmp(bench_ids[t].name,
+                            bench_suites[i].name) == 0
+                        || strcmp(bench_ids[t].name,
+                            bench_suites[i].cases[j].name) == 0)) {
+                    continue;
+                }
+
+                case_forperm(
+                        &bench_suites[i],
+                        &bench_suites[i].cases[j],
+                        bench_ids[t].defines,
+                        bench_ids[t].define_count,
+                        perm_run,
+                        NULL);
+            }
+        }
+    }
+}
+
+
+
+// option handling
+enum opt_flags {
+    OPT_HELP                     = 'h',
+    OPT_SUMMARY                  = 'Y',
+    OPT_LIST_SUITES              = 'l',
+    OPT_LIST_CASES               = 'L',
+    OPT_LIST_SUITE_PATHS         = 1,
+    OPT_LIST_CASE_PATHS          = 2,
+    OPT_LIST_DEFINES             = 3,
+    OPT_LIST_PERMUTATION_DEFINES = 4,
+    OPT_LIST_IMPLICIT_DEFINES    = 5,
+    OPT_LIST_GEOMETRIES          = 6,
+    OPT_DEFINE                   = 'D',
+    OPT_GEOMETRY                 = 'g',
+    OPT_STEP                     = 's',
+    OPT_DISK                     = 'd',
+    OPT_TRACE                    = 't',
+    OPT_READ_SLEEP               = 7,
+    OPT_PROG_SLEEP               = 8,
+    OPT_ERASE_SLEEP              = 9,
+};
+
+const char *short_opts = "hYlLD:g:s:d:t:";
+
+const struct option long_opts[] = {
+    {"help",             no_argument,       NULL, OPT_HELP},
+    {"summary",          no_argument,       NULL, OPT_SUMMARY},
+    {"list-suites",      no_argument,       NULL, OPT_LIST_SUITES},
+    {"list-cases",       no_argument,       NULL, OPT_LIST_CASES},
+    {"list-suite-paths", no_argument,       NULL, OPT_LIST_SUITE_PATHS},
+    {"list-case-paths",  no_argument,       NULL, OPT_LIST_CASE_PATHS},
+    {"list-defines",     no_argument,       NULL, OPT_LIST_DEFINES},
+    {"list-permutation-defines",
+                         no_argument,       NULL, OPT_LIST_PERMUTATION_DEFINES},
+    {"list-implicit-defines",
+                         no_argument,       NULL, OPT_LIST_IMPLICIT_DEFINES},
+    {"list-geometries",  no_argument,       NULL, OPT_LIST_GEOMETRIES},
+    {"define",           required_argument, NULL, OPT_DEFINE},
+    {"geometry",         required_argument, NULL, OPT_GEOMETRY},
+    {"step",             required_argument, NULL, OPT_STEP},
+    {"disk",             required_argument, NULL, OPT_DISK},
+    {"trace",            required_argument, NULL, OPT_TRACE},
+    {"read-sleep",       required_argument, NULL, OPT_READ_SLEEP},
+    {"prog-sleep",       required_argument, NULL, OPT_PROG_SLEEP},
+    {"erase-sleep",      required_argument, NULL, OPT_ERASE_SLEEP},
+    {NULL, 0, NULL, 0},
+};
+
+const char *const help_text[] = {
+    "Show this help message.",
+    "Show quick summary.",
+    "List bench suites.",
+    "List bench cases.",
+    "List the path for each bench suite.",
+    "List the path and line number for each bench case.",
+    "List all defines in this bench-runner.",
+    "List explicit defines in this bench-runner.",
+    "List implicit defines in this bench-runner.",
+    "List the available disk geometries.",
+    "Override a bench define.",
+    "Comma-separated list of disk geometries to bench. Defaults to d,e,E,n,N.",
+    "Comma-separated range of bench permutations to run (start,stop,step).",
+    "Redirect block device operations to this file.",
+    "Redirect trace output to this file.",
+    "Artificial read delay in seconds.",
+    "Artificial prog delay in seconds.",
+    "Artificial erase delay in seconds.",
+};
+
+int main(int argc, char **argv) {
+    void (*op)(void) = run;
+
+    size_t bench_override_capacity = 0;
+    size_t bench_geometry_capacity = 0;
+    size_t bench_id_capacity = 0;
+
+    // parse options
+    while (true) {
+        int c = getopt_long(argc, argv, short_opts, long_opts, NULL);
+        switch (c) {
+            // generate help message
+            case OPT_HELP: {
+                printf("usage: %s [options] [bench_id]\n", argv[0]);
+                printf("\n");
+
+                printf("options:\n");
+                size_t i = 0;
+                while (long_opts[i].name) {
+                    size_t indent;
+                    if (long_opts[i].has_arg == no_argument) {
+                        if (long_opts[i].val >= '0' && long_opts[i].val < 'z') {
+                            indent = printf("  -%c, --%s ",
+                                    long_opts[i].val,
+                                    long_opts[i].name);
+                        } else {
+                            indent = printf("  --%s ",
+                                    long_opts[i].name);
+                        }
+                    } else {
+                        if (long_opts[i].val >= '0' && long_opts[i].val < 'z') {
+                            indent = printf("  -%c %s, --%s %s ",
+                                    long_opts[i].val,
+                                    long_opts[i].name,
+                                    long_opts[i].name,
+                                    long_opts[i].name);
+                        } else {
+                            indent = printf("  --%s %s ",
+                                    long_opts[i].name,
+                                    long_opts[i].name);
+                        }
+                    }
+
+                    // a quick, hacky, byte-level method for text wrapping
+                    size_t len = strlen(help_text[i]);
+                    size_t j = 0;
+                    if (indent < 24) {
+                        printf("%*s %.80s\n",
+                                (int)(24-1-indent),
+                                "",
+                                &help_text[i][j]);
+                        j += 80;
+                    } else {
+                        printf("\n");
+                    }
+
+                    while (j < len) {
+                        printf("%24s%.80s\n", "", &help_text[i][j]);
+                        j += 80;
+                    }
+
+                    i += 1;
+                }
+
+                printf("\n");
+                exit(0);
+            }
+            // summary/list flags
+            case OPT_SUMMARY:
+                op = summary;
+                break;
+            case OPT_LIST_SUITES:
+                op = list_suites;
+                break;
+            case OPT_LIST_CASES:
+                op = list_cases;
+                break;
+            case OPT_LIST_SUITE_PATHS:
+                op = list_suite_paths;
+                break;
+            case OPT_LIST_CASE_PATHS:
+                op = list_case_paths;
+                break;
+            case OPT_LIST_DEFINES:
+                op = list_defines;
+                break;
+            case OPT_LIST_PERMUTATION_DEFINES:
+                op = list_permutation_defines;
+                break;
+            case OPT_LIST_IMPLICIT_DEFINES:
+                op = list_implicit_defines;
+                break;
+            case OPT_LIST_GEOMETRIES:
+                op = list_geometries;
+                break;
+            // configuration
+            case OPT_DEFINE: {
+                // allocate space
+                bench_override_t *override = mappend(
+                        (void**)&bench_overrides,
+                        sizeof(bench_override_t),
+                        &bench_override_count,
+                        &bench_override_capacity);
+
+                // parse into string key/intmax_t value, cannibalizing the
+                // arg in the process
+                char *sep = strchr(optarg, '=');
+                char *parsed = NULL;
+                if (!sep) {
+                    goto invalid_define;
+                }
+                *sep = '\0';
+                override->name = optarg;
+                optarg = sep+1;
+
+                // parse comma-separated permutations
+                {
+                    override->defines = NULL;
+                    override->permutations = 0;
+                    size_t override_capacity = 0;
+                    while (true) {
+                        optarg += strspn(optarg, " ");
+
+                        if (strncmp(optarg, "range", strlen("range")) == 0) {
+                            // range of values
+                            optarg += strlen("range");
+                            optarg += strspn(optarg, " ");
+                            if (*optarg != '(') {
+                                goto invalid_define;
+                            }
+                            optarg += 1;
+
+                            intmax_t start = strtoumax(optarg, &parsed, 0);
+                            intmax_t stop = -1;
+                            intmax_t step = 1;
+                            // allow empty string for start=0
+                            if (parsed == optarg) {
+                                start = 0;
+                            }
+                            optarg = parsed + strspn(parsed, " ");
+
+                            if (*optarg != ',' && *optarg != ')') {
+                                goto invalid_define;
+                            }
+
+                            if (*optarg == ',') {
+                                optarg += 1;
+                                stop = strtoumax(optarg, &parsed, 0);
+                                // allow empty string for stop=end
+                                if (parsed == optarg) {
+                                    stop = -1;
+                                }
+                                optarg = parsed + strspn(parsed, " ");
+
+                                if (*optarg != ',' && *optarg != ')') {
+                                    goto invalid_define;
+                                }
+
+                                if (*optarg == ',') {
+                                    optarg += 1;
+                                    step = strtoumax(optarg, &parsed, 0);
+                                    // allow empty string for stop=1
+                                    if (parsed == optarg) {
+                                        step = 1;
+                                    }
+                                    optarg = parsed + strspn(parsed, " ");
+
+                                    if (*optarg != ')') {
+                                        goto invalid_define;
+                                    }
+                                }
+                            } else {
+                                // single value = stop only
+                                stop = start;
+                                start = 0;
+                            }
+
+                            if (*optarg != ')') {
+                                goto invalid_define;
+                            }
+                            optarg += 1;
+
+                            // calculate the range of values
+                            assert(step != 0);
+                            for (intmax_t i = start;
+                                    (step < 0)
+                                        ? i > stop
+                                        : (uintmax_t)i < (uintmax_t)stop;
+                                    i += step) {
+                                *(intmax_t*)mappend(
+                                        (void**)&override->defines,
+                                        sizeof(intmax_t),
+                                        &override->permutations,
+                                        &override_capacity) = i;
+                            }
+                        } else if (*optarg != '\0') {
+                            // single value
+                            intmax_t define = strtoimax(optarg, &parsed, 0);
+                            if (parsed == optarg) {
+                                goto invalid_define;
+                            }
+                            optarg = parsed + strspn(parsed, " ");
+                            *(intmax_t*)mappend(
+                                    (void**)&override->defines,
+                                    sizeof(intmax_t),
+                                    &override->permutations,
+                                    &override_capacity) = define;
+                        } else {
+                            break;
+                        }
+
+                        if (*optarg == ',') {
+                            optarg += 1;
+                        }
+                    }
+                }
+                assert(override->permutations > 0);
+                break;
+
+invalid_define:
+                fprintf(stderr, "error: invalid define: %s\n", optarg);
+                exit(-1);
+            }
+            case OPT_GEOMETRY: {
+                // reset our geometry scenarios
+                if (bench_geometry_capacity > 0) {
+                    free((bench_geometry_t*)bench_geometries);
+                }
+                bench_geometries = NULL;
+                bench_geometry_count = 0;
+                bench_geometry_capacity = 0;
+
+                // parse the comma separated list of disk geometries
+                while (*optarg) {
+                    // allocate space
+                    bench_geometry_t *geometry = mappend(
+                            (void**)&bench_geometries,
+                            sizeof(bench_geometry_t),
+                            &bench_geometry_count,
+                            &bench_geometry_capacity);
+
+                    // parse the disk geometry
+                    optarg += strspn(optarg, " ");
+
+                    // named disk geometry
+                    size_t len = strcspn(optarg, " ,");
+                    for (size_t i = 0; builtin_geometries[i].long_name; i++) {
+                        if ((len == 1
+                                && *optarg == builtin_geometries[i].short_name)
+                                || (len == strlen(
+                                        builtin_geometries[i].long_name)
+                                    && memcmp(optarg,
+                                        builtin_geometries[i].long_name,
+                                        len) == 0))  {
+                            *geometry = builtin_geometries[i];
+                            optarg += len;
+                            goto geometry_next;
+                        }
+                    }
+
+                    // comma-separated read/prog/erase/count
+                    if (*optarg == '{') {
+                        lfs_size_t sizes[4];
+                        size_t count = 0;
+
+                        char *s = optarg + 1;
+                        while (count < 4) {
+                            char *parsed = NULL;
+                            sizes[count] = strtoumax(s, &parsed, 0);
+                            count += 1;
+
+                            s = parsed + strspn(parsed, " ");
+                            if (*s == ',') {
+                                s += 1;
+                                continue;
+                            } else if (*s == '}') {
+                                s += 1;
+                                break;
+                            } else {
+                                goto geometry_unknown;
+                            }
+                        }
+
+                        // allow implicit r=p and p=e for common geometries
+                        memset(geometry, 0, sizeof(bench_geometry_t));
+                        if (count >= 3) {
+                            geometry->defines[READ_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = BENCH_LIT(sizes[1]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[2]);
+                        } else if (count >= 2) {
+                            geometry->defines[PROG_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[1]);
+                        } else {
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                        }
+                        if (count >= 4) {
+                            geometry->defines[BLOCK_COUNT_i]
+                                    = BENCH_LIT(sizes[3]);
+                        }
+                        optarg = s;
+                        goto geometry_next;
+                    }
+
+                    // leb16-encoded read/prog/erase/count
+                    if (*optarg == ':') {
+                        lfs_size_t sizes[4];
+                        size_t count = 0;
+
+                        char *s = optarg + 1;
+                        while (true) {
+                            char *parsed = NULL;
+                            uintmax_t x = leb16_parse(s, &parsed);
+                            if (parsed == s || count >= 4) {
+                                break;
+                            }
+
+                            sizes[count] = x;
+                            count += 1;
+                            s = parsed;
+                        }
+
+                        // allow implicit r=p and p=e for common geometries
+                        memset(geometry, 0, sizeof(bench_geometry_t));
+                        if (count >= 3) {
+                            geometry->defines[READ_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = BENCH_LIT(sizes[1]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[2]);
+                        } else if (count >= 2) {
+                            geometry->defines[PROG_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[1]);
+                        } else {
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = BENCH_LIT(sizes[0]);
+                        }
+                        if (count >= 4) {
+                            geometry->defines[BLOCK_COUNT_i]
+                                    = BENCH_LIT(sizes[3]);
+                        }
+                        optarg = s;
+                        goto geometry_next;
+                    }
+
+geometry_unknown:
+                    // unknown scenario?
+                    fprintf(stderr, "error: unknown disk geometry: %s\n",
+                            optarg);
+                    exit(-1);
+
+geometry_next:
+                    optarg += strspn(optarg, " ");
+                    if (*optarg == ',') {
+                        optarg += 1;
+                    } else if (*optarg == '\0') {
+                        break;
+                    } else {
+                        goto geometry_unknown;
+                    }
+                }
+                break;
+            }
+            case OPT_STEP: {
+                char *parsed = NULL;
+                bench_step_start = strtoumax(optarg, &parsed, 0);
+                bench_step_stop = -1;
+                bench_step_step = 1;
+                // allow empty string for start=0
+                if (parsed == optarg) {
+                    bench_step_start = 0;
+                }
+                optarg = parsed + strspn(parsed, " ");
+
+                if (*optarg != ',' && *optarg != '\0') {
+                    goto step_unknown;
+                }
+
+                if (*optarg == ',') {
+                    optarg += 1;
+                    bench_step_stop = strtoumax(optarg, &parsed, 0);
+                    // allow empty string for stop=end
+                    if (parsed == optarg) {
+                        bench_step_stop = -1;
+                    }
+                    optarg = parsed + strspn(parsed, " ");
+
+                    if (*optarg != ',' && *optarg != '\0') {
+                        goto step_unknown;
+                    }
+
+                    if (*optarg == ',') {
+                        optarg += 1;
+                        bench_step_step = strtoumax(optarg, &parsed, 0);
+                        // allow empty string for stop=1
+                        if (parsed == optarg) {
+                            bench_step_step = 1;
+                        }
+                        optarg = parsed + strspn(parsed, " ");
+
+                        if (*optarg != '\0') {
+                            goto step_unknown;
+                        }
+                    }
+                } else {
+                    // single value = stop only
+                    bench_step_stop = bench_step_start;
+                    bench_step_start = 0;
+                }
+
+                break;
+step_unknown:
+                fprintf(stderr, "error: invalid step: %s\n", optarg);
+                exit(-1);
+            }
+            case OPT_DISK:
+                bench_disk_path = optarg;
+                break;
+            case OPT_TRACE:
+                bench_trace_path = optarg;
+                break;
+            case OPT_READ_SLEEP: {
+                char *parsed = NULL;
+                double read_sleep = strtod(optarg, &parsed);
+                if (parsed == optarg) {
+                    fprintf(stderr, "error: invalid read-sleep: %s\n", optarg);
+                    exit(-1);
+                }
+                bench_read_sleep = read_sleep*1.0e9;
+                break;
+            }
+            case OPT_PROG_SLEEP: {
+                char *parsed = NULL;
+                double prog_sleep = strtod(optarg, &parsed);
+                if (parsed == optarg) {
+                    fprintf(stderr, "error: invalid prog-sleep: %s\n", optarg);
+                    exit(-1);
+                }
+                bench_prog_sleep = prog_sleep*1.0e9;
+                break;
+            }
+            case OPT_ERASE_SLEEP: {
+                char *parsed = NULL;
+                double erase_sleep = strtod(optarg, &parsed);
+                if (parsed == optarg) {
+                    fprintf(stderr, "error: invalid erase-sleep: %s\n", optarg);
+                    exit(-1);
+                }
+                bench_erase_sleep = erase_sleep*1.0e9;
+                break;
+            }
+            // done parsing
+            case -1:
+                goto getopt_done;
+            // unknown arg, getopt prints a message for us
+            default:
+                exit(-1);
+        }
+    }
+getopt_done: ;
+
+    if (argc > optind) {
+        // reset our bench identifier list
+        bench_ids = NULL;
+        bench_id_count = 0;
+        bench_id_capacity = 0;
+    }
+
+    // parse bench identifier, if any, cannibalizing the arg in the process
+    for (; argc > optind; optind++) {
+        bench_define_t *defines = NULL;
+        size_t define_count = 0;
+
+        // parse name, can be suite or case
+        char *name = argv[optind];
+        char *defines_ = strchr(name, ':');
+        if (defines_) {
+            *defines_ = '\0';
+            defines_ += 1;
+        }
+
+        // remove optional path and .toml suffix
+        char *slash = strrchr(name, '/');
+        if (slash) {
+            name = slash+1;
+        }
+
+        size_t name_len = strlen(name);
+        if (name_len > 5 && strcmp(&name[name_len-5], ".toml") == 0) {
+            name[name_len-5] = '\0';
+        }
+
+        if (defines_) {
+            // parse defines
+            while (true) {
+                char *parsed;
+                size_t d = leb16_parse(defines_, &parsed);
+                intmax_t v = leb16_parse(parsed, &parsed);
+                if (parsed == defines_) {
+                    break;
+                }
+                defines_ = parsed;
+
+                if (d >= define_count) {
+                    // align to power of two to avoid any superlinear growth
+                    size_t ncount = 1 << lfs_npw2(d+1);
+                    defines = realloc(defines,
+                            ncount*sizeof(bench_define_t));
+                    memset(defines+define_count, 0,
+                            (ncount-define_count)*sizeof(bench_define_t));
+                    define_count = ncount;
+                }
+                defines[d] = BENCH_LIT(v);
+            }
+        }
+
+        // append to identifier list
+        *(bench_id_t*)mappend(
+                (void**)&bench_ids,
+                sizeof(bench_id_t),
+                &bench_id_count,
+                &bench_id_capacity) = (bench_id_t){
+            .name = name,
+            .defines = defines,
+            .define_count = define_count,
+        };
+    }
+
+    // do the thing
+    op();
+
+    // cleanup (need to be done for valgrind benching)
+    bench_define_cleanup();
+    if (bench_overrides) {
+        for (size_t i = 0; i < bench_override_count; i++) {
+            free((void*)bench_overrides[i].defines);
+        }
+        free((void*)bench_overrides);
+    }
+    if (bench_geometry_capacity) {
+        free((void*)bench_geometries);
+    }
+    if (bench_id_capacity) {
+        for (size_t i = 0; i < bench_id_count; i++) {
+            free((void*)bench_ids[i].defines);
+        }
+        free((void*)bench_ids);
+    }
+}

runners/bench_runner.h

@@ -0,0 +1,119 @@
+#ifndef BENCH_RUNNER_H
+#define BENCH_RUNNER_H
+
+
+// override LFS_TRACE
+void bench_trace(const char *fmt, ...);
+
+#define LFS_TRACE_(fmt, ...) \
+    bench_trace("%s:%d:trace: " fmt "%s\n", \
+        __FILE__, \
+        __LINE__, \
+        __VA_ARGS__)
+#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
+#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
+
+// provide BENCH_START/BENCH_STOP macros
+void bench_start(void);
+void bench_stop(void);
+
+#define BENCH_START() bench_start()
+#define BENCH_STOP() bench_stop()
+
+
+// note these are indirectly included in any generated files
+#include "bd/lfs_emubd.h"
+#include <stdio.h>
+
+// give source a chance to define feature macros
+#undef _FEATURES_H
+#undef _STDIO_H
+
+
+// generated bench configurations
+struct lfs_config;
+
+enum bench_flags {
+    BENCH_REENTRANT = 0x1,
+};
+typedef uint8_t bench_flags_t;
+
+typedef struct bench_define {
+    intmax_t (*cb)(void *data);
+    void *data;
+} bench_define_t;
+
+struct bench_case {
+    const char *name;
+    const char *path;
+    bench_flags_t flags;
+    size_t permutations;
+
+    const bench_define_t *defines;
+
+    bool (*filter)(void);
+    void (*run)(struct lfs_config *cfg);
+};
+
+struct bench_suite {
+    const char *name;
+    const char *path;
+    bench_flags_t flags;
+
+    const char *const *define_names;
+    size_t define_count;
+
+    const struct bench_case *cases;
+    size_t case_count;
+};
+
+
+// access generated bench defines
+intmax_t bench_define(size_t define);
+
+#define BENCH_DEFINE(i) bench_define(i)
+
+// a few preconfigured defines that control how benches run
+ 
+#define READ_SIZE_i          0
+#define PROG_SIZE_i          1
+#define BLOCK_SIZE_i         2
+#define BLOCK_COUNT_i        3
+#define CACHE_SIZE_i         4
+#define LOOKAHEAD_SIZE_i     5
+#define BLOCK_CYCLES_i       6
+#define ERASE_VALUE_i        7
+#define ERASE_CYCLES_i       8
+#define BADBLOCK_BEHAVIOR_i  9
+#define POWERLOSS_BEHAVIOR_i 10
+
+#define READ_SIZE           bench_define(READ_SIZE_i)
+#define PROG_SIZE           bench_define(PROG_SIZE_i)
+#define BLOCK_SIZE          bench_define(BLOCK_SIZE_i)
+#define BLOCK_COUNT         bench_define(BLOCK_COUNT_i)
+#define CACHE_SIZE          bench_define(CACHE_SIZE_i)
+#define LOOKAHEAD_SIZE      bench_define(LOOKAHEAD_SIZE_i)
+#define BLOCK_CYCLES        bench_define(BLOCK_CYCLES_i)
+#define ERASE_VALUE         bench_define(ERASE_VALUE_i)
+#define ERASE_CYCLES        bench_define(ERASE_CYCLES_i)
+#define BADBLOCK_BEHAVIOR   bench_define(BADBLOCK_BEHAVIOR_i)
+#define POWERLOSS_BEHAVIOR  bench_define(POWERLOSS_BEHAVIOR_i)
+
+#define BENCH_IMPLICIT_DEFINES \
+    BENCH_DEF(READ_SIZE,          PROG_SIZE) \
+    BENCH_DEF(PROG_SIZE,          BLOCK_SIZE) \
+    BENCH_DEF(BLOCK_SIZE,         0) \
+    BENCH_DEF(BLOCK_COUNT,        (1024*1024)/BLOCK_SIZE) \
+    BENCH_DEF(CACHE_SIZE,         lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \
+    BENCH_DEF(LOOKAHEAD_SIZE,     16) \
+    BENCH_DEF(BLOCK_CYCLES,       -1) \
+    BENCH_DEF(ERASE_VALUE,        0xff) \
+    BENCH_DEF(ERASE_CYCLES,       0) \
+    BENCH_DEF(BADBLOCK_BEHAVIOR,  LFS_EMUBD_BADBLOCK_PROGERROR) \
+    BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP)
+
+#define BENCH_GEOMETRY_DEFINE_COUNT 4
+#define BENCH_IMPLICIT_DEFINE_COUNT 11
+
+
+#endif

runners/test_runner.c

@@ -4,7 +4,7 @@
 #endif
 
 #include "runners/test_runner.h"
-#include "bd/lfs_testbd.h"
+#include "bd/lfs_emubd.h"
 
 #include <getopt.h>
 #include <sys/types.h>
@@ -46,7 +46,7 @@ void *mappend(void **p,
 // a quick self-terminating text-safe varint scheme
 static void leb16_print(uintmax_t x) {
     while (true) {
-        lfs_testbd_powercycles_t nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
+        char nibble = (x & 0xf) | (x > 0xf ? 0x10 : 0);
         printf("%c", (nibble < 10) ? '0'+nibble : 'a'+nibble-10);
         if (x <= 0xf) {
             break;
@@ -101,11 +101,11 @@ typedef struct test_powerloss {
     const char *long_name;
 
     void (*run)(
-            const lfs_testbd_powercycles_t *cycles,
+            const lfs_emubd_powercycles_t *cycles,
             size_t cycle_count,
             const struct test_suite *suite,
             const struct test_case *case_);
-    const lfs_testbd_powercycles_t *cycles;
+    const lfs_emubd_powercycles_t *cycles;
     size_t cycle_count;
 } test_powerloss_t;
 
@@ -113,7 +113,7 @@ typedef struct test_id {
     const char *name;
     const test_define_t *defines;
     size_t define_count;
-    const lfs_testbd_powercycles_t *cycles;
+    const lfs_emubd_powercycles_t *cycles;
     size_t cycle_count;
 } test_id_t;
 
@@ -141,17 +141,19 @@ typedef struct test_define_names {
 intmax_t test_define_lit(void *data) {
     return (intmax_t)data;
 }
-#define TEST_LIT(x) {test_define_lit, (void*)(uintptr_t)(x)}
 
+#define TEST_CONST(x) {test_define_lit, (void*)(uintptr_t)(x)}
+#define TEST_LIT(x) ((test_define_t)TEST_CONST(x))
 
-#define TEST_DEFINE(k, v) \
+
+#define TEST_DEF(k, v) \
     intmax_t test_define_##k(void *data) { \
         (void)data; \
         return v; \
     }
 
     TEST_IMPLICIT_DEFINES
-#undef TEST_DEFINE
+#undef TEST_DEF
 
 #define TEST_DEFINE_MAP_EXPLICIT    0
 #define TEST_DEFINE_MAP_OVERRIDE    1
@@ -163,11 +165,11 @@ intmax_t test_define_lit(void *data) {
 test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = {
     [TEST_DEFINE_MAP_IMPLICIT] = {
         (const test_define_t[TEST_IMPLICIT_DEFINE_COUNT]) {
-            #define TEST_DEFINE(k, v) \
+            #define TEST_DEF(k, v) \
                 [k##_i] = {test_define_##k, NULL},
 
                 TEST_IMPLICIT_DEFINES
-            #undef TEST_DEFINE
+            #undef TEST_DEF
         },
         TEST_IMPLICIT_DEFINE_COUNT,
     },
@@ -180,11 +182,11 @@ test_define_map_t test_define_maps[TEST_DEFINE_MAP_COUNT] = {
 test_define_names_t test_define_names[TEST_DEFINE_NAMES_COUNT] = {
     [TEST_DEFINE_NAMES_IMPLICIT] = {
         (const char *const[TEST_IMPLICIT_DEFINE_COUNT]){
-            #define TEST_DEFINE(k, v) \
+            #define TEST_DEF(k, v) \
                 [k##_i] = #k,
 
                 TEST_IMPLICIT_DEFINES
-            #undef TEST_DEFINE
+            #undef TEST_DEF
         },
         TEST_IMPLICIT_DEFINE_COUNT,
     },
@@ -318,7 +320,7 @@ void test_define_suite(const struct test_suite *suite) {
                 // define name match?
                 const char *name = test_define_name(d);
                 if (name && strcmp(name, test_overrides[i].name) == 0) {
-                    count = d+1;
+                    count = lfs_max(count, d+1);
                     permutations *= test_overrides[i].permutations;
                     break;
                 }
@@ -355,10 +357,9 @@ void test_define_suite(const struct test_suite *suite) {
                     // scatter the define permutations based on already
                     // seen permutations
                     for (size_t j = 0; j < permutations; j++) {
-                        test_override_defines[j*count + d]
-                                = (test_define_t)TEST_LIT(
-                                    test_overrides[i].defines[(j/p)
-                                        % test_overrides[i].permutations]);
+                        test_override_defines[j*count + d] = TEST_LIT(
+                                test_overrides[i].defines[(j/p)
+                                    % test_overrides[i].permutations]);
                     }
 
                     // keep track of how many permutations we've seen so far
@@ -426,9 +427,9 @@ const char *test_disk_path = NULL;
 const char *test_trace_path = NULL;
 FILE *test_trace_file = NULL;
 uint32_t test_trace_cycles = 0;
-lfs_testbd_sleep_t test_read_sleep = 0.0;
-lfs_testbd_sleep_t test_prog_sleep = 0.0;
-lfs_testbd_sleep_t test_erase_sleep = 0.0;
+lfs_emubd_sleep_t test_read_sleep = 0.0;
+lfs_emubd_sleep_t test_prog_sleep = 0.0;
+lfs_emubd_sleep_t test_erase_sleep = 0.0;
 
 
 // trace printing
@@ -485,10 +486,10 @@ void test_trace(const char *fmt, ...) {
 static void perm_printid(
         const struct test_suite *suite,
         const struct test_case *case_,
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count) {
     (void)suite;
-    // case[:permutation[:powercycles]]]
+    // case[:permutation[:powercycles]]
     printf("%s:", case_->name);
     for (size_t d = 0;
             d < lfs_max(
@@ -497,7 +498,7 @@ static void perm_printid(
             d++) {
         if (test_define_ispermutation(d)) {
             leb16_print(d);
-            leb16_print(test_define(d));
+            leb16_print(TEST_DEFINE(d));
         }
     }
 
@@ -511,7 +512,7 @@ static void perm_printid(
 }
 
 static void run_powerloss_cycles(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_);
@@ -522,7 +523,7 @@ static void case_forperm(
         const struct test_case *case_,
         const test_define_t *defines,
         size_t define_count,
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         void (*cb)(
             void *data,
@@ -834,7 +835,7 @@ static void list_defines_add(
         struct list_defines_defines *defines,
         size_t d) {
     const char *name = test_define_name(d);
-    intmax_t value = test_define(d);
+    intmax_t value = TEST_DEFINE(d);
 
     // define already in defines?
     for (size_t i = 0; i < defines->define_count; i++) {
@@ -1051,11 +1052,11 @@ static void list_implicit_defines(void) {
 // geometries to test
 
 const test_geometry_t builtin_geometries[] = {
-    {'d', "default", {{NULL}, TEST_LIT(16),   TEST_LIT(512),   {NULL}}},
-    {'e', "eeprom",  {{NULL}, TEST_LIT(1),    TEST_LIT(512),   {NULL}}},
-    {'E', "emmc",    {{NULL}, {NULL},         TEST_LIT(512),   {NULL}}},
-    {'n', "nor",     {{NULL}, TEST_LIT(1),    TEST_LIT(4096),  {NULL}}},
-    {'N', "nand",    {{NULL}, TEST_LIT(4096), TEST_LIT(32768), {NULL}}},
+    {'d', "default", {{NULL}, TEST_CONST(16),   TEST_CONST(512),   {NULL}}},
+    {'e', "eeprom",  {{NULL}, TEST_CONST(1),    TEST_CONST(512),   {NULL}}},
+    {'E', "emmc",    {{NULL}, {NULL},           TEST_CONST(512),   {NULL}}},
+    {'n', "nor",     {{NULL}, TEST_CONST(1),    TEST_CONST(4096),  {NULL}}},
+    {'N', "nand",    {{NULL}, TEST_CONST(4096), TEST_CONST(32768), {NULL}}},
     {0, NULL, {{NULL}, {NULL}, {NULL}, {NULL}}},
 };
 
@@ -1087,7 +1088,7 @@ static void list_geometries(void) {
 // scenarios to run tests under power-loss
 
 static void run_powerloss_none(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_) {
@@ -1096,14 +1097,14 @@ static void run_powerloss_none(
     (void)suite;
 
     // create block device and configuration
-    lfs_testbd_t bd;
+    lfs_emubd_t bd;
 
     struct lfs_config cfg = {
         .context            = &bd,
-        .read               = lfs_testbd_read,
-        .prog               = lfs_testbd_prog,
-        .erase              = lfs_testbd_erase,
-        .sync               = lfs_testbd_sync,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
         .read_size          = READ_SIZE,
         .prog_size          = PROG_SIZE,
         .block_size         = BLOCK_SIZE,
@@ -1113,7 +1114,7 @@ static void run_powerloss_none(
         .lookahead_size     = LOOKAHEAD_SIZE,
     };
 
-    struct lfs_testbd_config bdcfg = {
+    struct lfs_emubd_config bdcfg = {
         .erase_value        = ERASE_VALUE,
         .erase_cycles       = ERASE_CYCLES,
         .badblock_behavior  = BADBLOCK_BEHAVIOR,
@@ -1123,7 +1124,7 @@ static void run_powerloss_none(
         .erase_sleep        = test_erase_sleep,
     };
 
-    int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg);
+    int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg);
     if (err) {
         fprintf(stderr, "error: could not create block device: %d\n", err);
         exit(-1);
@@ -1141,7 +1142,7 @@ static void run_powerloss_none(
     printf("\n");
 
     // cleanup
-    err = lfs_testbd_destroy(&cfg);
+    err = lfs_emubd_destroy(&cfg);
     if (err) {
         fprintf(stderr, "error: could not destroy block device: %d\n", err);
         exit(-1);
@@ -1154,7 +1155,7 @@ static void powerloss_longjmp(void *c) {
 }
 
 static void run_powerloss_linear(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_) {
@@ -1163,16 +1164,16 @@ static void run_powerloss_linear(
     (void)suite;
 
     // create block device and configuration
-    lfs_testbd_t bd;
+    lfs_emubd_t bd;
     jmp_buf powerloss_jmp;
-    volatile lfs_testbd_powercycles_t i = 1;
+    volatile lfs_emubd_powercycles_t i = 1;
 
     struct lfs_config cfg = {
         .context            = &bd,
-        .read               = lfs_testbd_read,
-        .prog               = lfs_testbd_prog,
-        .erase              = lfs_testbd_erase,
-        .sync               = lfs_testbd_sync,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
         .read_size          = READ_SIZE,
         .prog_size          = PROG_SIZE,
         .block_size         = BLOCK_SIZE,
@@ -1182,7 +1183,7 @@ static void run_powerloss_linear(
         .lookahead_size     = LOOKAHEAD_SIZE,
     };
 
-    struct lfs_testbd_config bdcfg = {
+    struct lfs_emubd_config bdcfg = {
         .erase_value        = ERASE_VALUE,
         .erase_cycles       = ERASE_CYCLES,
         .badblock_behavior  = BADBLOCK_BEHAVIOR,
@@ -1196,7 +1197,7 @@ static void run_powerloss_linear(
         .powerloss_data     = &powerloss_jmp,
     };
 
-    int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg);
+    int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg);
     if (err) {
         fprintf(stderr, "error: could not create block device: %d\n", err);
         exit(-1);
@@ -1218,13 +1219,13 @@ static void run_powerloss_linear(
         printf("powerloss ");
         perm_printid(suite, case_, NULL, 0);
         printf(":");
-        for (lfs_testbd_powercycles_t j = 1; j <= i; j++) {
+        for (lfs_emubd_powercycles_t j = 1; j <= i; j++) {
             leb16_print(j);
         }
         printf("\n");
 
         i += 1;
-        lfs_testbd_setpowercycles(&cfg, i);
+        lfs_emubd_setpowercycles(&cfg, i);
     }
 
     printf("finished ");
@@ -1232,7 +1233,7 @@ static void run_powerloss_linear(
     printf("\n");
 
     // cleanup
-    err = lfs_testbd_destroy(&cfg);
+    err = lfs_emubd_destroy(&cfg);
     if (err) {
         fprintf(stderr, "error: could not destroy block device: %d\n", err);
         exit(-1);
@@ -1240,7 +1241,7 @@ static void run_powerloss_linear(
 }
 
 static void run_powerloss_exponential(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_) {
@@ -1249,16 +1250,16 @@ static void run_powerloss_exponential(
     (void)suite;
 
     // create block device and configuration
-    lfs_testbd_t bd;
+    lfs_emubd_t bd;
     jmp_buf powerloss_jmp;
-    volatile lfs_testbd_powercycles_t i = 1;
+    volatile lfs_emubd_powercycles_t i = 1;
 
     struct lfs_config cfg = {
         .context            = &bd,
-        .read               = lfs_testbd_read,
-        .prog               = lfs_testbd_prog,
-        .erase              = lfs_testbd_erase,
-        .sync               = lfs_testbd_sync,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
         .read_size          = READ_SIZE,
         .prog_size          = PROG_SIZE,
         .block_size         = BLOCK_SIZE,
@@ -1268,7 +1269,7 @@ static void run_powerloss_exponential(
         .lookahead_size     = LOOKAHEAD_SIZE,
     };
 
-    struct lfs_testbd_config bdcfg = {
+    struct lfs_emubd_config bdcfg = {
         .erase_value        = ERASE_VALUE,
         .erase_cycles       = ERASE_CYCLES,
         .badblock_behavior  = BADBLOCK_BEHAVIOR,
@@ -1282,7 +1283,7 @@ static void run_powerloss_exponential(
         .powerloss_data     = &powerloss_jmp,
     };
 
-    int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg);
+    int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg);
     if (err) {
         fprintf(stderr, "error: could not create block device: %d\n", err);
         exit(-1);
@@ -1304,13 +1305,13 @@ static void run_powerloss_exponential(
         printf("powerloss ");
         perm_printid(suite, case_, NULL, 0);
         printf(":");
-        for (lfs_testbd_powercycles_t j = 1; j <= i; j *= 2) {
+        for (lfs_emubd_powercycles_t j = 1; j <= i; j *= 2) {
             leb16_print(j);
         }
         printf("\n");
 
         i *= 2;
-        lfs_testbd_setpowercycles(&cfg, i);
+        lfs_emubd_setpowercycles(&cfg, i);
     }
 
     printf("finished ");
@@ -1318,7 +1319,7 @@ static void run_powerloss_exponential(
     printf("\n");
 
     // cleanup
-    err = lfs_testbd_destroy(&cfg);
+    err = lfs_emubd_destroy(&cfg);
     if (err) {
         fprintf(stderr, "error: could not destroy block device: %d\n", err);
         exit(-1);
@@ -1326,23 +1327,23 @@ static void run_powerloss_exponential(
 }
 
 static void run_powerloss_cycles(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_) {
     (void)suite;
 
     // create block device and configuration
-    lfs_testbd_t bd;
+    lfs_emubd_t bd;
     jmp_buf powerloss_jmp;
     volatile size_t i = 0;
 
     struct lfs_config cfg = {
         .context            = &bd,
-        .read               = lfs_testbd_read,
-        .prog               = lfs_testbd_prog,
-        .erase              = lfs_testbd_erase,
-        .sync               = lfs_testbd_sync,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
         .read_size          = READ_SIZE,
         .prog_size          = PROG_SIZE,
         .block_size         = BLOCK_SIZE,
@@ -1352,7 +1353,7 @@ static void run_powerloss_cycles(
         .lookahead_size     = LOOKAHEAD_SIZE,
     };
 
-    struct lfs_testbd_config bdcfg = {
+    struct lfs_emubd_config bdcfg = {
         .erase_value        = ERASE_VALUE,
         .erase_cycles       = ERASE_CYCLES,
         .badblock_behavior  = BADBLOCK_BEHAVIOR,
@@ -1366,7 +1367,7 @@ static void run_powerloss_cycles(
         .powerloss_data     = &powerloss_jmp,
     };
 
-    int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg);
+    int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg);
     if (err) {
         fprintf(stderr, "error: could not create block device: %d\n", err);
         exit(-1);
@@ -1391,7 +1392,7 @@ static void run_powerloss_cycles(
         printf("\n");
 
         i += 1;
-        lfs_testbd_setpowercycles(&cfg,
+        lfs_emubd_setpowercycles(&cfg,
                 (i < cycle_count) ? cycles[i] : 0);
     }
 
@@ -1400,7 +1401,7 @@ static void run_powerloss_cycles(
     printf("\n");
 
     // cleanup
-    err = lfs_testbd_destroy(&cfg);
+    err = lfs_emubd_destroy(&cfg);
     if (err) {
         fprintf(stderr, "error: could not destroy block device: %d\n", err);
         exit(-1);
@@ -1410,13 +1411,13 @@ static void run_powerloss_cycles(
 struct powerloss_exhaustive_state {
     struct lfs_config *cfg;
 
-    lfs_testbd_t *branches;
+    lfs_emubd_t *branches;
     size_t branch_count;
     size_t branch_capacity;
 };
 
 struct powerloss_exhaustive_cycles {
-    lfs_testbd_powercycles_t *cycles;
+    lfs_emubd_powercycles_t *cycles;
     size_t cycle_count;
     size_t cycle_capacity;
 };
@@ -1424,9 +1425,9 @@ struct powerloss_exhaustive_cycles {
 static void powerloss_exhaustive_branch(void *c) {
     struct powerloss_exhaustive_state *state = c;
     // append to branches
-    lfs_testbd_t *branch = mappend(
+    lfs_emubd_t *branch = mappend(
             (void**)&state->branches,
-            sizeof(lfs_testbd_t),
+            sizeof(lfs_emubd_t),
             &state->branch_count,
             &state->branch_capacity);
     if (!branch) {
@@ -1435,14 +1436,14 @@ static void powerloss_exhaustive_branch(void *c) {
     }
 
     // create copy-on-write copy
-    int err = lfs_testbd_copy(state->cfg, branch);
+    int err = lfs_emubd_copy(state->cfg, branch);
     if (err) {
         fprintf(stderr, "error: exhaustive: could not create bd copy\n");
         exit(-1);
     }
 
     // also trigger on next power cycle
-    lfs_testbd_setpowercycles(state->cfg, 1);
+    lfs_emubd_setpowercycles(state->cfg, 1);
 }
 
 static void run_powerloss_exhaustive_layer(
@@ -1450,7 +1451,7 @@ static void run_powerloss_exhaustive_layer(
         const struct test_suite *suite,
         const struct test_case *case_,
         struct lfs_config *cfg,
-        struct lfs_testbd_config *bdcfg,
+        struct lfs_emubd_config *bdcfg,
         size_t depth) {
     (void)suite;
 
@@ -1463,14 +1464,14 @@ static void run_powerloss_exhaustive_layer(
 
     // run through the test without additional powerlosses, collecting possible
     // branches as we do so
-    lfs_testbd_setpowercycles(state.cfg, depth > 0 ? 1 : 0);
+    lfs_emubd_setpowercycles(state.cfg, depth > 0 ? 1 : 0);
     bdcfg->powerloss_data = &state;
 
     // run the tests
     case_->run(cfg);
 
     // aggressively clean up memory here to try to keep our memory usage low
-    int err = lfs_testbd_destroy(cfg);
+    int err = lfs_emubd_destroy(cfg);
     if (err) {
         fprintf(stderr, "error: could not destroy block device: %d\n", err);
         exit(-1);
@@ -1479,9 +1480,9 @@ static void run_powerloss_exhaustive_layer(
     // recurse into each branch
     for (size_t i = 0; i < state.branch_count; i++) {
         // first push and print the branch
-        lfs_testbd_powercycles_t *cycle = mappend(
+        lfs_emubd_powercycles_t *cycle = mappend(
                 (void**)&cycles->cycles,
-                sizeof(lfs_testbd_powercycles_t),
+                sizeof(lfs_emubd_powercycles_t),
                 &cycles->cycle_count,
                 &cycles->cycle_capacity);
         if (!cycle) {
@@ -1509,7 +1510,7 @@ static void run_powerloss_exhaustive_layer(
 }
 
 static void run_powerloss_exhaustive(
-        const lfs_testbd_powercycles_t *cycles,
+        const lfs_emubd_powercycles_t *cycles,
         size_t cycle_count,
         const struct test_suite *suite,
         const struct test_case *case_) {
@@ -1517,14 +1518,14 @@ static void run_powerloss_exhaustive(
     (void)suite;
 
     // create block device and configuration
-    lfs_testbd_t bd;
+    lfs_emubd_t bd;
 
     struct lfs_config cfg = {
         .context            = &bd,
-        .read               = lfs_testbd_read,
-        .prog               = lfs_testbd_prog,
-        .erase              = lfs_testbd_erase,
-        .sync               = lfs_testbd_sync,
+        .read               = lfs_emubd_read,
+        .prog               = lfs_emubd_prog,
+        .erase              = lfs_emubd_erase,
+        .sync               = lfs_emubd_sync,
         .read_size          = READ_SIZE,
         .prog_size          = PROG_SIZE,
         .block_size         = BLOCK_SIZE,
@@ -1534,7 +1535,7 @@ static void run_powerloss_exhaustive(
         .lookahead_size     = LOOKAHEAD_SIZE,
     };
 
-    struct lfs_testbd_config bdcfg = {
+    struct lfs_emubd_config bdcfg = {
         .erase_value        = ERASE_VALUE,
         .erase_cycles       = ERASE_CYCLES,
         .badblock_behavior  = BADBLOCK_BEHAVIOR,
@@ -1547,7 +1548,7 @@ static void run_powerloss_exhaustive(
         .powerloss_data     = NULL,
     };
 
-    int err = lfs_testbd_createcfg(&cfg, test_disk_path, &bdcfg);
+    int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg);
     if (err) {
         fprintf(stderr, "error: could not create block device: %d\n", err);
         exit(-1);
@@ -1956,6 +1957,7 @@ int main(int argc, char **argv) {
                             if (parsed == optarg) {
                                 goto invalid_define;
                             }
+                            optarg = parsed + strspn(parsed, " ");
                             *(intmax_t*)mappend(
                                     (void**)&override->defines,
                                     sizeof(intmax_t),
@@ -1965,7 +1967,6 @@ int main(int argc, char **argv) {
                             break;
                         }
 
-                        optarg = parsed + strspn(parsed, " ");
                         if (*optarg == ',') {
                             optarg += 1;
                         }
@@ -2041,24 +2042,24 @@ invalid_define:
                         // allow implicit r=p and p=e for common geometries
                         memset(geometry, 0, sizeof(test_geometry_t));
                         if (count >= 3) {
-                            geometry->defines[0]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
-                            geometry->defines[1]
-                                    = (test_define_t)TEST_LIT(sizes[1]);
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[2]);
+                            geometry->defines[READ_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = TEST_LIT(sizes[1]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[2]);
                         } else if (count >= 2) {
-                            geometry->defines[1]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[1]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[1]);
                         } else {
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
                         }
                         if (count >= 4) {
-                            geometry->defines[3]
-                                    = (test_define_t)TEST_LIT(sizes[3]);
+                            geometry->defines[BLOCK_COUNT_i]
+                                    = TEST_LIT(sizes[3]);
                         }
                         optarg = s;
                         goto geometry_next;
@@ -2085,24 +2086,24 @@ invalid_define:
                         // allow implicit r=p and p=e for common geometries
                         memset(geometry, 0, sizeof(test_geometry_t));
                         if (count >= 3) {
-                            geometry->defines[0]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
-                            geometry->defines[1]
-                                    = (test_define_t)TEST_LIT(sizes[1]);
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[2]);
+                            geometry->defines[READ_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = TEST_LIT(sizes[1]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[2]);
                         } else if (count >= 2) {
-                            geometry->defines[1]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[1]);
+                            geometry->defines[PROG_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[1]);
                         } else {
-                            geometry->defines[2]
-                                    = (test_define_t)TEST_LIT(sizes[0]);
+                            geometry->defines[BLOCK_SIZE_i]
+                                    = TEST_LIT(sizes[0]);
                         }
                         if (count >= 4) {
-                            geometry->defines[3]
-                                    = (test_define_t)TEST_LIT(sizes[3]);
+                            geometry->defines[BLOCK_COUNT_i]
+                                    = TEST_LIT(sizes[3]);
                         }
                         optarg = s;
                         goto geometry_next;
@@ -2165,16 +2166,16 @@ geometry_next:
 
                     // comma-separated permutation
                     if (*optarg == '{') {
-                        lfs_testbd_powercycles_t *cycles = NULL;
+                        lfs_emubd_powercycles_t *cycles = NULL;
                         size_t cycle_count = 0;
                         size_t cycle_capacity = 0;
 
                         char *s = optarg + 1;
                         while (true) {
                             char *parsed = NULL;
-                            *(lfs_testbd_powercycles_t*)mappend(
+                            *(lfs_emubd_powercycles_t*)mappend(
                                     (void**)&cycles,
-                                    sizeof(lfs_testbd_powercycles_t),
+                                    sizeof(lfs_emubd_powercycles_t),
                                     &cycle_count,
                                     &cycle_capacity)
                                     = strtoumax(s, &parsed, 0);
@@ -2202,7 +2203,7 @@ geometry_next:
 
                     // leb16-encoded permutation
                     if (*optarg == ':') {
-                        lfs_testbd_powercycles_t *cycles = NULL;
+                        lfs_emubd_powercycles_t *cycles = NULL;
                         size_t cycle_count = 0;
                         size_t cycle_capacity = 0;
 
@@ -2214,9 +2215,9 @@ geometry_next:
                                 break;
                             }
 
-                            *(lfs_testbd_powercycles_t*)mappend(
+                            *(lfs_emubd_powercycles_t*)mappend(
                                     (void**)&cycles,
-                                    sizeof(lfs_testbd_powercycles_t),
+                                    sizeof(lfs_emubd_powercycles_t),
                                     &cycle_count,
                                     &cycle_capacity) = x;
                             s = parsed;
@@ -2374,7 +2375,7 @@ getopt_done: ;
     for (; argc > optind; optind++) {
         test_define_t *defines = NULL;
         size_t define_count = 0;
-        lfs_testbd_powercycles_t *cycles = NULL;
+        lfs_emubd_powercycles_t *cycles = NULL;
         size_t cycle_count = 0;
 
         // parse name, can be suite or case
@@ -2422,7 +2423,7 @@ getopt_done: ;
                             (ncount-define_count)*sizeof(test_define_t));
                     define_count = ncount;
                 }
-                defines[d] = (test_define_t)TEST_LIT(v);
+                defines[d] = TEST_LIT(v);
             }
 
             if (cycles_) {
@@ -2430,9 +2431,9 @@ getopt_done: ;
                 size_t cycle_capacity = 0;
                 while (*cycles_ != '\0') {
                     char *parsed = NULL;
-                    *(lfs_testbd_powercycles_t*)mappend(
+                    *(lfs_emubd_powercycles_t*)mappend(
                             (void**)&cycles,
-                            sizeof(lfs_testbd_powercycles_t),
+                            sizeof(lfs_emubd_powercycles_t),
                             &cycle_count,
                             &cycle_capacity)
                             = leb16_parse(cycles_, &parsed);

runners/test_runner.h

@@ -11,11 +11,11 @@ void test_trace(const char *fmt, ...);
         __LINE__, \
         __VA_ARGS__)
 #define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
-#define LFS_TESTBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
+#define LFS_EMUBD_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
 
 
 // note these are indirectly included in any generated files
-#include "bd/lfs_testbd.h"
+#include "bd/lfs_emubd.h"
 #include <stdio.h>
 
 // give source a chance to define feature macros
@@ -62,9 +62,10 @@ struct test_suite {
 
 
 // access generated test defines
-//intmax_t test_predefine(size_t define);
 intmax_t test_define(size_t define);
 
+#define TEST_DEFINE(i) test_define(i)
+
 // a few preconfigured defines that control how tests run
  
 #define READ_SIZE_i          0
@@ -79,33 +80,33 @@ intmax_t test_define(size_t define);
 #define BADBLOCK_BEHAVIOR_i  9
 #define POWERLOSS_BEHAVIOR_i 10
 
-#define READ_SIZE           test_define(READ_SIZE_i)
-#define PROG_SIZE           test_define(PROG_SIZE_i)
-#define BLOCK_SIZE          test_define(BLOCK_SIZE_i)
-#define BLOCK_COUNT         test_define(BLOCK_COUNT_i)
-#define CACHE_SIZE          test_define(CACHE_SIZE_i)
-#define LOOKAHEAD_SIZE      test_define(LOOKAHEAD_SIZE_i)
-#define BLOCK_CYCLES        test_define(BLOCK_CYCLES_i)
-#define ERASE_VALUE         test_define(ERASE_VALUE_i)
-#define ERASE_CYCLES        test_define(ERASE_CYCLES_i)
-#define BADBLOCK_BEHAVIOR   test_define(BADBLOCK_BEHAVIOR_i)
-#define POWERLOSS_BEHAVIOR  test_define(POWERLOSS_BEHAVIOR_i)
+#define READ_SIZE           TEST_DEFINE(READ_SIZE_i)
+#define PROG_SIZE           TEST_DEFINE(PROG_SIZE_i)
+#define BLOCK_SIZE          TEST_DEFINE(BLOCK_SIZE_i)
+#define BLOCK_COUNT         TEST_DEFINE(BLOCK_COUNT_i)
+#define CACHE_SIZE          TEST_DEFINE(CACHE_SIZE_i)
+#define LOOKAHEAD_SIZE      TEST_DEFINE(LOOKAHEAD_SIZE_i)
+#define BLOCK_CYCLES        TEST_DEFINE(BLOCK_CYCLES_i)
+#define ERASE_VALUE         TEST_DEFINE(ERASE_VALUE_i)
+#define ERASE_CYCLES        TEST_DEFINE(ERASE_CYCLES_i)
+#define BADBLOCK_BEHAVIOR   TEST_DEFINE(BADBLOCK_BEHAVIOR_i)
+#define POWERLOSS_BEHAVIOR  TEST_DEFINE(POWERLOSS_BEHAVIOR_i)
 
 #define TEST_IMPLICIT_DEFINES \
-    TEST_DEFINE(READ_SIZE,          PROG_SIZE) \
-    TEST_DEFINE(PROG_SIZE,          BLOCK_SIZE) \
-    TEST_DEFINE(BLOCK_SIZE,         0) \
-    TEST_DEFINE(BLOCK_COUNT,        (1024*1024)/BLOCK_SIZE) \
-    TEST_DEFINE(CACHE_SIZE,         lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \
-    TEST_DEFINE(LOOKAHEAD_SIZE,     16) \
-    TEST_DEFINE(BLOCK_CYCLES,       -1) \
-    TEST_DEFINE(ERASE_VALUE,        0xff) \
-    TEST_DEFINE(ERASE_CYCLES,       0) \
-    TEST_DEFINE(BADBLOCK_BEHAVIOR,  LFS_TESTBD_BADBLOCK_PROGERROR) \
-    TEST_DEFINE(POWERLOSS_BEHAVIOR, LFS_TESTBD_POWERLOSS_NOOP)
+    TEST_DEF(READ_SIZE,          PROG_SIZE) \
+    TEST_DEF(PROG_SIZE,          BLOCK_SIZE) \
+    TEST_DEF(BLOCK_SIZE,         0) \
+    TEST_DEF(BLOCK_COUNT,        (1024*1024)/BLOCK_SIZE) \
+    TEST_DEF(CACHE_SIZE,         lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \
+    TEST_DEF(LOOKAHEAD_SIZE,     16) \
+    TEST_DEF(BLOCK_CYCLES,       -1) \
+    TEST_DEF(ERASE_VALUE,        0xff) \
+    TEST_DEF(ERASE_CYCLES,       0) \
+    TEST_DEF(BADBLOCK_BEHAVIOR,  LFS_EMUBD_BADBLOCK_PROGERROR) \
+    TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP)
 
-#define TEST_GEOMETRY_DEFINE_COUNT 4
 #define TEST_IMPLICIT_DEFINE_COUNT 11
+#define TEST_GEOMETRY_DEFINE_COUNT 4
 
 
 #endif

scripts/bench.py

@@ -0,0 +1,1355 @@
+#!/usr/bin/env python3
+#
+# Script to compile and runs benches.
+#
+# Example:
+# ./scripts/bench.py runners/bench_runner -b
+#
+# Copyright (c) 2022, The littlefs authors.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+import collections as co
+import csv
+import errno
+import glob
+import itertools as it
+import math as m
+import os
+import pty
+import re
+import shlex
+import shutil
+import signal
+import subprocess as sp
+import threading as th
+import time
+import toml
+
+
+RUNNER_PATH = 'runners/bench_runner'
+HEADER_PATH = 'runners/bench_runner.h'
+
+
+def openio(path, mode='r', buffering=-1, nb=False):
+    if path == '-':
+        if mode == 'r':
+            return os.fdopen(os.dup(sys.stdin.fileno()), 'r', buffering)
+        else:
+            return os.fdopen(os.dup(sys.stdout.fileno()), 'w', buffering)
+    elif nb and 'a' in mode:
+        return os.fdopen(os.open(
+                path,
+                os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NONBLOCK,
+                0o666),
+            mode,
+            buffering)
+    else:
+        return open(path, mode, buffering)
+
+class BenchCase:
+    # create a BenchCase object from a config
+    def __init__(self, config, args={}):
+        self.name = config.pop('name')
+        self.path = config.pop('path')
+        self.suite = config.pop('suite')
+        self.lineno = config.pop('lineno', None)
+        self.if_ = config.pop('if', None)
+        if isinstance(self.if_, bool):
+            self.if_ = 'true' if self.if_ else 'false'
+        self.code = config.pop('code')
+        self.code_lineno = config.pop('code_lineno', None)
+        self.in_ = config.pop('in',
+            config.pop('suite_in', None))
+
+        # figure out defines and build possible permutations
+        self.defines = set()
+        self.permutations = []
+
+        # defines can be a dict or a list or dicts
+        suite_defines = config.pop('suite_defines', {})
+        if not isinstance(suite_defines, list):
+            suite_defines = [suite_defines]
+        defines = config.pop('defines', {})
+        if not isinstance(defines, list):
+            defines = [defines]
+
+        def csplit(v):
+            # split commas but only outside of parens
+            parens = 0
+            i_ = 0
+            for i in range(len(v)):
+                if v[i] == ',' and parens == 0:
+                    yield v[i_:i]
+                    i_ = i+1
+                elif v[i] in '([{':
+                    parens += 1
+                elif v[i] in '}])':
+                    parens -= 1
+            if v[i_:].strip():
+                yield v[i_:]
+
+        def parse_define(v):
+            # a define entry can be a list
+            if isinstance(v, list):
+                for v_ in v:
+                    yield from parse_define(v_)
+            # or a string
+            elif isinstance(v, str):
+                # which can be comma-separated values, with optional
+                # range statements. This matches the runtime define parser in
+                # the runner itself.
+                for v_ in csplit(v):
+                    m = re.search(r'\brange\b\s*\('
+                        '(?P<start>[^,\s]*)'
+                        '\s*(?:,\s*(?P<stop>[^,\s]*)'
+                        '\s*(?:,\s*(?P<step>[^,\s]*)\s*)?)?\)',
+                        v_)
+                    if m:
+                        start = (int(m.group('start'), 0)
+                            if m.group('start') else 0)
+                        stop = (int(m.group('stop'), 0)
+                            if m.group('stop') else None)
+                        step = (int(m.group('step'), 0)
+                            if m.group('step') else 1)
+                        if m.lastindex <= 1:
+                            start, stop = 0, start
+                        for x in range(start, stop, step):
+                            yield from parse_define('%s(%d)%s' % (
+                                v_[:m.start()], x, v_[m.end():]))
+                    else:
+                        yield v_
+            # or a literal value
+            else:
+                yield v
+
+        # build possible permutations
+        for suite_defines_ in suite_defines:
+            self.defines |= suite_defines_.keys()
+            for defines_ in defines:
+                self.defines |= defines_.keys()
+                self.permutations.extend(dict(perm) for perm in it.product(*(
+                    [(k, v) for v in parse_define(vs)]
+                    for k, vs in sorted((suite_defines_ | defines_).items()))))
+
+        for k in config.keys():
+            print('%swarning:%s in %s, found unused key %r' % (
+                '\x1b[01;33m' if args['color'] else '',
+                '\x1b[m' if args['color'] else '',
+                self.name,
+                k),
+                file=sys.stderr)
+
+
+class BenchSuite:
+    # create a BenchSuite object from a toml file
+    def __init__(self, path, args={}):
+        self.path = path
+        self.name = os.path.basename(path)
+        if self.name.endswith('.toml'):
+            self.name = self.name[:-len('.toml')]
+
+        # load toml file and parse bench cases
+        with open(self.path) as f:
+            # load benches
+            config = toml.load(f)
+
+            # find line numbers
+            f.seek(0)
+            case_linenos = []
+            code_linenos = []
+            for i, line in enumerate(f):
+                match = re.match(
+                    '(?P<case>\[\s*cases\s*\.\s*(?P<name>\w+)\s*\])'
+                        '|' '(?P<code>code\s*=)',
+                    line)
+                if match and match.group('case'):
+                    case_linenos.append((i+1, match.group('name')))
+                elif match and match.group('code'):
+                    code_linenos.append(i+2)
+
+            # sort in case toml parsing did not retain order
+            case_linenos.sort()
+
+            cases = config.pop('cases')
+            for (lineno, name), (nlineno, _) in it.zip_longest(
+                    case_linenos, case_linenos[1:],
+                    fillvalue=(float('inf'), None)):
+                code_lineno = min(
+                    (l for l in code_linenos if l >= lineno and l < nlineno),
+                    default=None)
+                cases[name]['lineno'] = lineno
+                cases[name]['code_lineno'] = code_lineno
+
+            self.if_ = config.pop('if', None)
+            if isinstance(self.if_, bool):
+                self.if_ = 'true' if self.if_ else 'false'
+
+            self.code = config.pop('code', None)
+            self.code_lineno = min(
+                (l for l in code_linenos
+                    if not case_linenos or l < case_linenos[0][0]),
+                default=None)
+
+            # a couple of these we just forward to all cases
+            defines = config.pop('defines', {})
+            in_ = config.pop('in', None)
+
+            self.cases = []
+            for name, case in sorted(cases.items(),
+                    key=lambda c: c[1].get('lineno')):
+                self.cases.append(BenchCase(config={
+                    'name': name,
+                    'path': path + (':%d' % case['lineno']
+                        if 'lineno' in case else ''),
+                    'suite': self.name,
+                    'suite_defines': defines,
+                    'suite_in': in_,
+                    **case},
+                    args=args))
+
+            # combine per-case defines
+            self.defines = set.union(*(
+                set(case.defines) for case in self.cases))
+
+        for k in config.keys():
+            print('%swarning:%s in %s, found unused key %r' % (
+                '\x1b[01;33m' if args['color'] else '',
+                '\x1b[m' if args['color'] else '',
+                self.name,
+                k),
+                file=sys.stderr)
+
+
+
+def compile(bench_paths, **args):
+    # find .toml files
+    paths = []
+    for path in bench_paths:
+        if os.path.isdir(path):
+            path = path + '/*.toml'
+
+        for path in glob.glob(path):
+            paths.append(path)
+
+    if not paths:
+        print('no bench suites found in %r?' % bench_paths)
+        sys.exit(-1)
+
+    # load the suites
+    suites = [BenchSuite(path, args) for path in paths]
+    suites.sort(key=lambda s: s.name)
+
+    # check for name conflicts, these will cause ambiguity problems later
+    # when running benches
+    seen = {}
+    for suite in suites:
+        if suite.name in seen:
+            print('%swarning:%s conflicting suite %r, %s and %s' % (
+                '\x1b[01;33m' if args['color'] else '',
+                '\x1b[m' if args['color'] else '',
+                suite.name,
+                suite.path,
+                seen[suite.name].path),
+                file=sys.stderr)
+        seen[suite.name] = suite
+
+        for case in suite.cases:
+            # only allow conflicts if a case and its suite share a name
+            if case.name in seen and not (
+                    isinstance(seen[case.name], BenchSuite)
+                    and seen[case.name].cases == [case]):
+                print('%swarning:%s conflicting case %r, %s and %s' % (
+                    '\x1b[01;33m' if args['color'] else '',
+                    '\x1b[m' if args['color'] else '',
+                    case.name,
+                    case.path,
+                    seen[case.name].path),
+                    file=sys.stderr)
+            seen[case.name] = case
+
+    # we can only compile one bench suite at a time
+    if not args.get('source'):
+        if len(suites) > 1:
+            print('more than one bench suite for compilation? (%r)' % bench_paths)
+            sys.exit(-1)
+
+        suite = suites[0]
+
+    # write generated bench source
+    if 'output' in args:
+        with openio(args['output'], 'w') as f:
+            _write = f.write
+            def write(s):
+                f.lineno += s.count('\n')
+                _write(s)
+            def writeln(s=''):
+                f.lineno += s.count('\n') + 1
+                _write(s)
+                _write('\n')
+            f.lineno = 1
+            f.write = write
+            f.writeln = writeln
+
+            f.writeln("// Generated by %s:" % sys.argv[0])
+            f.writeln("//")
+            f.writeln("// %s" % ' '.join(sys.argv))
+            f.writeln("//")
+            f.writeln()
+
+            # include bench_runner.h in every generated file
+            f.writeln("#include \"%s\"" % args['include'])
+            f.writeln()
+
+            # write out generated functions, this can end up in different
+            # files depending on the "in" attribute
+            #
+            # note it's up to the specific generated file to declare
+            # the bench defines
+            def write_case_functions(f, suite, case):
+                    # create case define functions
+                    if case.defines:
+                        # deduplicate defines by value to try to reduce the
+                        # number of functions we generate
+                        define_cbs = {}
+                        for i, defines in enumerate(case.permutations):
+                            for k, v in sorted(defines.items()):
+                                if v not in define_cbs:
+                                    name = ('__bench__%s__%s__%s__%d'
+                                        % (suite.name, case.name, k, i))
+                                    define_cbs[v] = name
+                                    f.writeln('intmax_t %s('
+                                        '__attribute__((unused)) '
+                                        'void *data) {' % name)
+                                    f.writeln(4*' '+'return %s;' % v)
+                                    f.writeln('}')
+                                    f.writeln()
+                        f.writeln('const bench_define_t '
+                            '__bench__%s__%s__defines[]['
+                            'BENCH_IMPLICIT_DEFINE_COUNT+%d] = {'
+                            % (suite.name, case.name, len(suite.defines)))
+                        for defines in case.permutations:
+                            f.writeln(4*' '+'{')
+                            for k, v in sorted(defines.items()):
+                                f.writeln(8*' '+'[%-24s] = {%s, NULL},' % (
+                                    k+'_i', define_cbs[v]))
+                            f.writeln(4*' '+'},')
+                        f.writeln('};')
+                        f.writeln()
+
+                    # create case filter function
+                    if suite.if_ is not None or case.if_ is not None:
+                        f.writeln('bool __bench__%s__%s__filter(void) {'
+                            % (suite.name, case.name))
+                        f.writeln(4*' '+'return %s;'
+                            % ' && '.join('(%s)' % if_
+                                for if_ in [suite.if_, case.if_]
+                                if if_ is not None))
+                        f.writeln('}')
+                        f.writeln()
+
+                    # create case run function
+                    f.writeln('void __bench__%s__%s__run('
+                        '__attribute__((unused)) struct lfs_config *cfg) {'
+                        % (suite.name, case.name))
+                    f.writeln(4*' '+'// bench case %s' % case.name)
+                    if case.code_lineno is not None:
+                        f.writeln(4*' '+'#line %d "%s"'
+                            % (case.code_lineno, suite.path))
+                    f.write(case.code)
+                    if case.code_lineno is not None:
+                        f.writeln(4*' '+'#line %d "%s"'
+                            % (f.lineno+1, args['output']))
+                    f.writeln('}')
+                    f.writeln()
+
+            if not args.get('source'):
+                if suite.code is not None:
+                    if suite.code_lineno is not None:
+                        f.writeln('#line %d "%s"'
+                            % (suite.code_lineno, suite.path))
+                    f.write(suite.code)
+                    if suite.code_lineno is not None:
+                        f.writeln('#line %d "%s"'
+                            % (f.lineno+1, args['output']))
+                    f.writeln()
+
+                if suite.defines:
+                    for i, define in enumerate(sorted(suite.defines)):
+                        f.writeln('#ifndef %s' % define)
+                        f.writeln('#define %-24s '
+                            'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i))
+                        f.writeln('#define %-24s '
+                            'BENCH_DEFINE(%s)' % (define, define+'_i'))
+                        f.writeln('#endif')
+                    f.writeln()
+
+                # create case functions
+                for case in suite.cases:
+                    if case.in_ is None:
+                        write_case_functions(f, suite, case)
+                    else:
+                        if case.defines:
+                            f.writeln('extern const bench_define_t '
+                                '__bench__%s__%s__defines[]['
+                                'BENCH_IMPLICIT_DEFINE_COUNT+%d];'
+                                % (suite.name, case.name, len(suite.defines)))
+                        if suite.if_ is not None or case.if_ is not None:
+                            f.writeln('extern bool __bench__%s__%s__filter('
+                                'void);'
+                                % (suite.name, case.name))
+                        f.writeln('extern void __bench__%s__%s__run('
+                            'struct lfs_config *cfg);'
+                            % (suite.name, case.name))
+                        f.writeln()
+
+                # create suite struct
+                #
+                # note we place this in the custom bench_suites section with
+                # minimum alignment, otherwise GCC ups the alignment to
+                # 32-bytes for some reason
+                f.writeln('__attribute__((section("_bench_suites"), '
+                    'aligned(1)))')
+                f.writeln('const struct bench_suite __bench__%s__suite = {'
+                    % suite.name)
+                f.writeln(4*' '+'.name = "%s",' % suite.name)
+                f.writeln(4*' '+'.path = "%s",' % suite.path)
+                f.writeln(4*' '+'.flags = 0,')
+                if suite.defines:
+                    # create suite define names
+                    f.writeln(4*' '+'.define_names = (const char *const['
+                        'BENCH_IMPLICIT_DEFINE_COUNT+%d]){' % (
+                        len(suite.defines)))
+                    for k in sorted(suite.defines):
+                        f.writeln(8*' '+'[%-24s] = "%s",' % (k+'_i', k))
+                    f.writeln(4*' '+'},')
+                    f.writeln(4*' '+'.define_count = '
+                        'BENCH_IMPLICIT_DEFINE_COUNT+%d,' % len(suite.defines))
+                f.writeln(4*' '+'.cases = (const struct bench_case[]){')
+                for case in suite.cases:
+                    # create case structs
+                    f.writeln(8*' '+'{')
+                    f.writeln(12*' '+'.name = "%s",' % case.name)
+                    f.writeln(12*' '+'.path = "%s",' % case.path)
+                    f.writeln(12*' '+'.flags = 0,')
+                    f.writeln(12*' '+'.permutations = %d,'
+                        % len(case.permutations))
+                    if case.defines:
+                        f.writeln(12*' '+'.defines '
+                            '= (const bench_define_t*)__bench__%s__%s__defines,'
+                            % (suite.name, case.name))
+                    if suite.if_ is not None or case.if_ is not None:
+                        f.writeln(12*' '+'.filter = __bench__%s__%s__filter,'
+                            % (suite.name, case.name))
+                    f.writeln(12*' '+'.run = __bench__%s__%s__run,'
+                        % (suite.name, case.name))
+                    f.writeln(8*' '+'},')
+                f.writeln(4*' '+'},')
+                f.writeln(4*' '+'.case_count = %d,' % len(suite.cases))
+                f.writeln('};')
+                f.writeln()
+
+            else:
+                # copy source
+                f.writeln('#line 1 "%s"' % args['source'])
+                with open(args['source']) as sf:
+                    shutil.copyfileobj(sf, f)
+                f.writeln()
+
+                # write any internal benches
+                for suite in suites:
+                    for case in suite.cases:
+                        if (case.in_ is not None
+                                and os.path.normpath(case.in_)
+                                    == os.path.normpath(args['source'])):
+                            # write defines, but note we need to undef any
+                            # new defines since we're in someone else's file
+                            if suite.defines:
+                                for i, define in enumerate(
+                                        sorted(suite.defines)):
+                                    f.writeln('#ifndef %s' % define)
+                                    f.writeln('#define %-24s '
+                                        'BENCH_IMPLICIT_DEFINE_COUNT+%d' % (
+                                        define+'_i', i))
+                                    f.writeln('#define %-24s '
+                                        'BENCH_DEFINE(%s)' % (
+                                        define, define+'_i'))
+                                    f.writeln('#define '
+                                        '__BENCH__%s__NEEDS_UNDEF' % (
+                                        define))
+                                    f.writeln('#endif')
+                                f.writeln()
+
+                            write_case_functions(f, suite, case)
+
+                            if suite.defines:
+                                for define in sorted(suite.defines):
+                                    f.writeln('#ifdef __BENCH__%s__NEEDS_UNDEF'
+                                        % define)
+                                    f.writeln('#undef __BENCH__%s__NEEDS_UNDEF'
+                                        % define)
+                                    f.writeln('#undef %s' % define)
+                                    f.writeln('#undef %s' % (define+'_i'))
+                                    f.writeln('#endif')
+                                f.writeln()
+
+def find_runner(runner, **args):
+    cmd = runner.copy()
+
+    # run under some external command?
+    if args.get('exec'):
+        cmd[:0] = args['exec']
+
+    # run under valgrind?
+    if args.get('valgrind'):
+        cmd[:0] = filter(None, [
+            'valgrind',
+            '--leak-check=full',
+            '--track-origins=yes',
+            '--error-exitcode=4',
+            '-q'])
+
+    # other context
+    if args.get('geometry'):
+        cmd.append('-g%s' % args['geometry'])
+    if args.get('disk'):
+        cmd.append('-d%s' % args['disk'])
+    if args.get('trace'):
+        cmd.append('-t%s' % args['trace'])
+    if args.get('read_sleep'):
+        cmd.append('--read-sleep=%s' % args['read_sleep'])
+    if args.get('prog_sleep'):
+        cmd.append('--prog-sleep=%s' % args['prog_sleep'])
+    if args.get('erase_sleep'):
+        cmd.append('--erase-sleep=%s' % args['erase_sleep'])
+
+    # defines?
+    if args.get('define'):
+        for define in args.get('define'):
+            cmd.append('-D%s' % define)
+
+    return cmd
+
+def list_(runner, bench_ids=[], **args):
+    cmd = find_runner(runner, **args) + bench_ids
+    if args.get('summary'):          cmd.append('--summary')
+    if args.get('list_suites'):      cmd.append('--list-suites')
+    if args.get('list_cases'):       cmd.append('--list-cases')
+    if args.get('list_suite_paths'): cmd.append('--list-suite-paths')
+    if args.get('list_case_paths'):  cmd.append('--list-case-paths')
+    if args.get('list_defines'):     cmd.append('--list-defines')
+    if args.get('list_permutation_defines'):
+                                     cmd.append('--list-permutation-defines')
+    if args.get('list_implicit_defines'):
+                                     cmd.append('--list-implicit-defines')
+    if args.get('list_geometries'):  cmd.append('--list-geometries')
+
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    return sp.call(cmd)
+
+
+def find_perms(runner_, ids=[], **args):
+    case_suites = {}
+    expected_case_perms = co.defaultdict(lambda: 0)
+    expected_perms = 0
+    total_perms = 0
+
+    # query cases from the runner
+    cmd = runner_ + ['--list-cases'] + ids
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    pattern = re.compile(
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<flags>[^\s]+)'
+            '\s+(?P<filtered>\d+)/(?P<perms>\d+)')
+    # skip the first line
+    for line in it.islice(proc.stdout, 1, None):
+        m = pattern.match(line)
+        if m:
+            filtered = int(m.group('filtered'))
+            perms = int(m.group('perms'))
+            expected_case_perms[m.group('case')] += filtered
+            expected_perms += filtered
+            total_perms += perms
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    # get which suite each case belongs to via paths
+    cmd = runner_ + ['--list-case-paths'] + ids
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    pattern = re.compile(
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
+    # skip the first line
+    for line in it.islice(proc.stdout, 1, None):
+        m = pattern.match(line)
+        if m:
+            path = m.group('path')
+            # strip path/suffix here
+            suite = os.path.basename(path)
+            if suite.endswith('.toml'):
+                suite = suite[:-len('.toml')]
+            case_suites[m.group('case')] = suite
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    # figure out expected suite perms
+    expected_suite_perms = co.defaultdict(lambda: 0)
+    for case, suite in case_suites.items():
+        expected_suite_perms[suite] += expected_case_perms[case]
+
+    return (
+        case_suites,
+        expected_suite_perms,
+        expected_case_perms,
+        expected_perms,
+        total_perms)
+
+def find_path(runner_, id, **args):
+    path = None
+    # query from runner
+    cmd = runner_ + ['--list-case-paths', id]
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    pattern = re.compile(
+        '^(?P<case>[^\s]+)'
+            '\s+(?P<path>[^:]+):(?P<lineno>\d+)')
+    # skip the first line
+    for line in it.islice(proc.stdout, 1, None):
+        m = pattern.match(line)
+        if m and path is None:
+            path_ = m.group('path')
+            lineno = int(m.group('lineno'))
+            path = (path_, lineno)
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    return path
+
+def find_defines(runner_, id, **args):
+    # query permutation defines from runner
+    cmd = runner_ + ['--list-permutation-defines', id]
+    if args.get('verbose'):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.PIPE,
+        stderr=sp.PIPE if not args.get('verbose') else None,
+        universal_newlines=True,
+        errors='replace',
+        close_fds=False)
+    defines = co.OrderedDict()
+    pattern = re.compile('^(?P<define>\w+)=(?P<value>.+)')
+    for line in proc.stdout:
+        m = pattern.match(line)
+        if m:
+            define = m.group('define')
+            value = m.group('value')
+            defines[define] = value
+    proc.wait()
+    if proc.returncode != 0:
+        if not args.get('verbose'):
+            for line in proc.stderr:
+                sys.stdout.write(line)
+        sys.exit(-1)
+
+    return defines
+
+
+# Thread-safe CSV writer
+class BenchOutput:
+    def __init__(self, path, head=None, tail=None):
+        self.f = openio(path, 'w+', 1)
+        self.lock = th.Lock()
+        self.head = head or []
+        self.tail = tail or []
+        self.writer = csv.DictWriter(self.f, self.head + self.tail)
+        self.rows = []
+
+    def close(self):
+        self.f.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        self.f.close()
+
+    def writerow(self, row):
+        with self.lock:
+            self.rows.append(row)
+            if all(k in self.head or k in self.tail for k in row.keys()):
+                # can simply append
+                self.writer.writerow(row)
+            else:
+                # need to rewrite the file
+                self.head.extend(row.keys() - (self.head + self.tail))
+                self.f.seek(0)
+                self.f.truncate()
+                self.writer = csv.DictWriter(self.f, self.head + self.tail)
+                self.writer.writeheader()
+                for row in self.rows:
+                    self.writer.writerow(row)
+
+# A bench failure
+class BenchFailure(Exception):
+    def __init__(self, id, returncode, stdout, assert_=None):
+        self.id = id
+        self.returncode = returncode
+        self.stdout = stdout
+        self.assert_ = assert_
+
+def run_stage(name, runner_, ids, output_, **args):
+    # get expected suite/case/perm counts
+    (case_suites,
+        expected_suite_perms,
+        expected_case_perms,
+        expected_perms,
+        total_perms) = find_perms(runner_, ids, **args)
+
+    passed_suite_perms = co.defaultdict(lambda: 0)
+    passed_case_perms = co.defaultdict(lambda: 0)
+    passed_perms = 0
+    read = 0
+    prog = 0
+    erased = 0
+    failures = []
+    killed = False
+
+    pattern = re.compile('^(?:'
+            '(?P<op>running|finished|skipped|powerloss)'
+                ' (?P<id>(?P<case>[^:]+)[^\s]*)'
+                '(?: (?P<read>\d+))?'
+                '(?: (?P<prog>\d+))?'
+                '(?: (?P<erased>\d+))?'
+            '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
+                ' *(?P<message>.*)'
+        ')$')
+    locals = th.local()
+    children = set()
+
+    def run_runner(runner_, ids=[]):
+        nonlocal passed_suite_perms
+        nonlocal passed_case_perms
+        nonlocal passed_perms
+        nonlocal read
+        nonlocal prog
+        nonlocal erased
+        nonlocal locals
+
+        # run the benches!
+        cmd = runner_ + ids
+        if args.get('verbose'):
+            print(' '.join(shlex.quote(c) for c in cmd))
+
+        mpty, spty = pty.openpty()
+        proc = sp.Popen(cmd, stdout=spty, stderr=spty, close_fds=False)
+        os.close(spty)
+        children.add(proc)
+        mpty = os.fdopen(mpty, 'r', 1)
+        stdout = None
+
+        last_id = None
+        last_stdout = []
+        last_assert = None
+        try:
+            while True:
+                # parse a line for state changes
+                try:
+                    line = mpty.readline()
+                except OSError as e:
+                    if e.errno == errno.EIO:
+                        break
+                    raise
+                if not line:
+                    break
+                last_stdout.append(line)
+                if args.get('stdout'):
+                    try:
+                        if not stdout:
+                            stdout = openio(args['stdout'], 'a', 1, nb=True)
+                        stdout.write(line)
+                    except OSError as e:
+                        if e.errno not in [
+                                errno.ENXIO,
+                                errno.EPIPE,
+                                errno.EAGAIN]:
+                            raise
+                        stdout = None
+                if args.get('verbose'):
+                    sys.stdout.write(line)
+
+                m = pattern.match(line)
+                if m:
+                    op = m.group('op') or m.group('op_')
+                    if op == 'running':
+                        locals.seen_perms += 1
+                        last_id = m.group('id')
+                        last_stdout = []
+                        last_assert = None
+                    elif op == 'finished':
+                        case = m.group('case')
+                        suite = case_suites[case]
+                        read_ = int(m.group('read'))
+                        prog_ = int(m.group('prog'))
+                        erased_ = int(m.group('erased'))
+                        passed_suite_perms[suite] += 1
+                        passed_case_perms[case] += 1
+                        passed_perms += 1
+                        read += read_
+                        prog += prog_
+                        erased += erased_
+                        if output_:
+                            # get defines and write to csv
+                            defines = find_defines(
+                                runner_, m.group('id'), **args)
+                            output_.writerow({
+                                'suite': suite,
+                                'case': case,
+                                'bench_read': read_,
+                                'bench_prog': prog_,
+                                'bench_erased': erased_,
+                                **defines})
+                    elif op == 'skipped':
+                        locals.seen_perms += 1
+                    elif op == 'assert':
+                        last_assert = (
+                            m.group('path'),
+                            int(m.group('lineno')),
+                            m.group('message'))
+                        # go ahead and kill the process, aborting takes a while
+                        if args.get('keep_going'):
+                            proc.kill()
+        except KeyboardInterrupt:
+            raise BenchFailure(last_id, 1, last_stdout)
+        finally:
+            children.remove(proc)
+            mpty.close()
+
+        proc.wait()
+        if proc.returncode != 0:
+            raise BenchFailure(
+                last_id,
+                proc.returncode,
+                last_stdout,
+                last_assert)
+
+    def run_job(runner_, ids=[], start=None, step=None):
+        nonlocal failures
+        nonlocal killed
+        nonlocal locals
+
+        start = start or 0
+        step = step or 1
+        while start < total_perms:
+            job_runner = runner_.copy()
+            if args.get('isolate') or args.get('valgrind'):
+                job_runner.append('-s%s,%s,%s' % (start, start+step, step))
+            else:
+                job_runner.append('-s%s,,%s' % (start, step))
+
+            try:
+                # run the benches
+                locals.seen_perms = 0
+                run_runner(job_runner, ids)
+                assert locals.seen_perms > 0
+                start += locals.seen_perms*step
+
+            except BenchFailure as failure:
+                # keep track of failures
+                if output_:
+                    case, _ = failure.id.split(':', 1)
+                    suite = case_suites[case]
+                    # get defines and write to csv
+                    defines = find_defines(runner_, failure.id, **args)
+                    output_.writerow({
+                        'suite': suite,
+                        'case': case,
+                        **defines})
+
+                # race condition for multiple failures?
+                if failures and not args.get('keep_going'):
+                    break
+
+                failures.append(failure)
+
+                if args.get('keep_going') and not killed:
+                    # resume after failed bench
+                    assert locals.seen_perms > 0
+                    start += locals.seen_perms*step
+                    continue
+                else:
+                    # stop other benches
+                    killed = True
+                    for child in children.copy():
+                        child.kill()
+                    break
+
+
+    # parallel jobs?
+    runners = []
+    if 'jobs' in args:
+        for job in range(args['jobs']):
+            runners.append(th.Thread(
+                target=run_job, args=(runner_, ids, job, args['jobs']),
+                daemon=True))
+    else:
+        runners.append(th.Thread(
+            target=run_job, args=(runner_, ids, None, None),
+            daemon=True))
+
+    def print_update(done):
+        if not args.get('verbose') and (args['color'] or done):
+            sys.stdout.write('%s%srunning %s%s:%s %s%s' % (
+                '\r\x1b[K' if args['color'] else '',
+                '\x1b[?7l' if not done else '',
+                ('\x1b[34m' if not failures else '\x1b[31m')
+                    if args['color'] else '',
+                name,
+                '\x1b[m' if args['color'] else '',
+                ', '.join(filter(None, [
+                    '%d/%d suites' % (
+                        sum(passed_suite_perms[k] == v
+                            for k, v in expected_suite_perms.items()),
+                        len(expected_suite_perms))
+                        if (not args.get('by_suites')
+                            and not args.get('by_cases')) else None,
+                    '%d/%d cases' % (
+                        sum(passed_case_perms[k] == v
+                            for k, v in expected_case_perms.items()),
+                        len(expected_case_perms))
+                        if not args.get('by_cases') else None,
+                    '%d/%d perms' % (passed_perms, expected_perms),
+                    '%s%d/%d failures%s' % (
+                            '\x1b[31m' if args['color'] else '',
+                            len(failures),
+                            expected_perms,
+                            '\x1b[m' if args['color'] else '')
+                        if failures else None])),
+                '\x1b[?7h' if not done else '\n'))
+            sys.stdout.flush()
+
+    for r in runners:
+        r.start()
+
+    try:
+        while any(r.is_alive() for r in runners):
+            time.sleep(0.01)
+            print_update(False)
+    except KeyboardInterrupt:
+        # this is handled by the runner threads, we just
+        # need to not abort here
+        killed = True
+    finally:
+        print_update(True)
+
+    for r in runners:
+        r.join()
+
+    return (
+        expected_perms,
+        passed_perms,
+        read,
+        prog,
+        erased,
+        failures,
+        killed)
+
+
+def run(runner, bench_ids=[], **args):
+    # query runner for benches
+    runner_ = find_runner(runner, **args)
+    print('using runner: %s' % ' '.join(shlex.quote(c) for c in runner_))
+    (_,
+        expected_suite_perms,
+        expected_case_perms,
+        expected_perms,
+        total_perms) = find_perms(runner_, bench_ids, **args)
+    print('found %d suites, %d cases, %d/%d permutations' % (
+        len(expected_suite_perms),
+        len(expected_case_perms),
+        expected_perms,
+        total_perms))
+    print()
+
+    # truncate and open logs here so they aren't disconnected between benches
+    stdout = None
+    if args.get('stdout'):
+        stdout = openio(args['stdout'], 'w', 1)
+    trace = None
+    if args.get('trace'):
+        trace = openio(args['trace'], 'w', 1)
+    output = None
+    if args.get('output'):
+        output = BenchOutput(args['output'],
+            ['suite', 'case'],
+            ['bench_read', 'bench_prog', 'bench_erased'])
+
+    # measure runtime
+    start = time.time()
+
+    # spawn runners
+    expected = 0
+    passed = 0
+    read = 0
+    prog = 0
+    erased = 0
+    failures = []
+    for by in (expected_case_perms.keys() if args.get('by_cases')
+            else expected_suite_perms.keys() if args.get('by_suites')
+            else [None]):
+        # spawn jobs for stage
+        (expected_,
+            passed_,
+            read_,
+            prog_,
+            erased_,
+            failures_,
+            killed) = run_stage(
+                by or 'benches',
+                runner_,
+                [by] if by is not None else bench_ids,
+                output,
+                **args)
+        # collect passes/failures
+        expected += expected_
+        passed += passed_
+        read += read_
+        prog += prog_
+        erased += erased_
+        failures.extend(failures_)
+        if (failures and not args.get('keep_going')) or killed:
+            break
+
+    stop = time.time()
+
+    if stdout:
+        stdout.close()
+    if trace:
+        trace.close()
+    if output:
+        output.close()
+
+    # show summary
+    print()
+    print('%sdone:%s %s' % (
+        ('\x1b[34m' if not failures else '\x1b[31m')
+            if args['color'] else '',
+        '\x1b[m' if args['color'] else '',
+        ', '.join(filter(None, [
+            '%d read' % read,
+            '%d prog' % prog,
+            '%d erased' % erased,
+            'in %.2fs' % (stop-start)]))))
+    print()
+
+    # print each failure
+    for failure in failures:
+        assert failure.id is not None, '%s broken? %r' % (
+            ' '.join(shlex.quote(c) for c in runner_),
+            failure)
+
+        # get some extra info from runner
+        path, lineno = find_path(runner_, failure.id, **args)
+        defines = find_defines(runner_, failure.id, **args)
+
+        # show summary of failure
+        print('%s%s:%d:%sfailure:%s %s%s failed' % (
+            '\x1b[01m' if args['color'] else '',
+            path, lineno,
+            '\x1b[01;31m' if args['color'] else '',
+            '\x1b[m' if args['color'] else '',
+            failure.id,
+            ' (%s)' % ', '.join('%s=%s' % (k,v) for k,v in defines.items())
+                if defines else ''))
+
+        if failure.stdout:
+            stdout = failure.stdout
+            if failure.assert_ is not None:
+                stdout = stdout[:-1]
+            for line in stdout[-args.get('context', 5):]:
+                sys.stdout.write(line)
+
+        if failure.assert_ is not None:
+            path, lineno, message = failure.assert_
+            print('%s%s:%d:%sassert:%s %s' % (
+                '\x1b[01m' if args['color'] else '',
+                path, lineno,
+                '\x1b[01;31m' if args['color'] else '',
+                '\x1b[m' if args['color'] else '',
+                message))
+            with open(path) as f:
+                line = next(it.islice(f, lineno-1, None)).strip('\n')
+                print(line)
+        print()
+
+    # drop into gdb?
+    if failures and (args.get('gdb')
+            or args.get('gdb_case')
+            or args.get('gdb_main')):
+        failure = failures[0]
+        cmd = runner_ + [failure.id]
+
+        if args.get('gdb_main'):
+            cmd[:0] = ['gdb',
+                '-ex', 'break main',
+                '-ex', 'run',
+                '--args']
+        elif args.get('gdb_case'):
+            path, lineno = find_path(runner_, failure.id, **args)
+            cmd[:0] = ['gdb',
+                '-ex', 'break %s:%d' % (path, lineno),
+                '-ex', 'run',
+                '--args']
+        elif failure.assert_ is not None:
+            cmd[:0] = ['gdb',
+                '-ex', 'run',
+                '-ex', 'frame function raise',
+                '-ex', 'up 2',
+                '--args']
+        else:
+            cmd[:0] = ['gdb',
+                '-ex', 'run',
+                '--args']
+
+        # exec gdb interactively
+        if args.get('verbose'):
+            print(' '.join(shlex.quote(c) for c in cmd))
+        os.execvp(cmd[0], cmd)
+
+    return 1 if failures else 0
+
+
+def main(**args):
+    # figure out what color should be
+    if args.get('color') == 'auto':
+        args['color'] = sys.stdout.isatty()
+    elif args.get('color') == 'always':
+        args['color'] = True
+    else:
+        args['color'] = False
+
+    if args.get('compile'):
+        return compile(**args)
+    elif (args.get('summary')
+            or args.get('list_suites')
+            or args.get('list_cases')
+            or args.get('list_suite_paths')
+            or args.get('list_case_paths')
+            or args.get('list_defines')
+            or args.get('list_permutation_defines')
+            or args.get('list_implicit_defines')
+            or args.get('list_geometries')):
+        return list_(**args)
+    else:
+        return run(**args)
+
+
+if __name__ == "__main__":
+    import argparse
+    import sys
+    argparse.ArgumentParser._handle_conflict_ignore = lambda *_: None
+    argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None
+    parser = argparse.ArgumentParser(
+        description="Build and run benches.",
+        conflict_handler='ignore')
+    parser.add_argument(
+        '-v', '--verbose',
+        action='store_true',
+        help="Output commands that run behind the scenes.")
+    parser.add_argument(
+        '--color',
+        choices=['never', 'always', 'auto'],
+        default='auto',
+        help="When to use terminal colors. Defaults to 'auto'.")
+
+    # bench flags
+    bench_parser = parser.add_argument_group('bench options')
+    bench_parser.add_argument(
+        'runner',
+        nargs='?',
+        type=lambda x: x.split(),
+        help="Bench runner to use for benching. Defaults to %r." % RUNNER_PATH)
+    bench_parser.add_argument(
+        'bench_ids',
+        nargs='*',
+        help="Description of benches to run.")
+    bench_parser.add_argument(
+        '-Y', '--summary',
+        action='store_true',
+        help="Show quick summary.")
+    bench_parser.add_argument(
+        '-l', '--list-suites',
+        action='store_true',
+        help="List bench suites.")
+    bench_parser.add_argument(
+        '-L', '--list-cases',
+        action='store_true',
+        help="List bench cases.")
+    bench_parser.add_argument(
+        '--list-suite-paths',
+        action='store_true',
+        help="List the path for each bench suite.")
+    bench_parser.add_argument(
+        '--list-case-paths',
+        action='store_true',
+        help="List the path and line number for each bench case.")
+    bench_parser.add_argument(
+        '--list-defines',
+        action='store_true',
+        help="List all defines in this bench-runner.")
+    bench_parser.add_argument(
+        '--list-permutation-defines',
+        action='store_true',
+        help="List explicit defines in this bench-runner.")
+    bench_parser.add_argument(
+        '--list-implicit-defines',
+        action='store_true',
+        help="List implicit defines in this bench-runner.")
+    bench_parser.add_argument(
+        '--list-geometries',
+        action='store_true',
+        help="List the available disk geometries.")
+    bench_parser.add_argument(
+        '-D', '--define',
+        action='append',
+        help="Override a bench define.")
+    bench_parser.add_argument(
+        '-g', '--geometry',
+        help="Comma-separated list of disk geometries to bench. "
+            "Defaults to d,e,E,n,N.")
+    bench_parser.add_argument(
+        '-d', '--disk',
+        help="Direct block device operations to this file.")
+    bench_parser.add_argument(
+        '-t', '--trace',
+        help="Direct trace output to this file.")
+    bench_parser.add_argument(
+        '-O', '--stdout',
+        help="Direct stdout to this file. Note stderr is already merged here.")
+    bench_parser.add_argument(
+        '-o', '--output',
+        help="CSV file to store results.")
+    bench_parser.add_argument(
+        '--read-sleep',
+        help="Artificial read delay in seconds.")
+    bench_parser.add_argument(
+        '--prog-sleep',
+        help="Artificial prog delay in seconds.")
+    bench_parser.add_argument(
+        '--erase-sleep',
+        help="Artificial erase delay in seconds.")
+    bench_parser.add_argument(
+        '-j', '--jobs',
+        nargs='?',
+        type=lambda x: int(x, 0),
+        const=len(os.sched_getaffinity(0)),
+        help="Number of parallel runners to run.")
+    bench_parser.add_argument(
+        '-k', '--keep-going',
+        action='store_true',
+        help="Don't stop on first error.")
+    bench_parser.add_argument(
+        '-i', '--isolate',
+        action='store_true',
+        help="Run each bench permutation in a separate process.")
+    bench_parser.add_argument(
+        '-b', '--by-suites',
+        action='store_true',
+        help="Step through benches by suite.")
+    bench_parser.add_argument(
+        '-B', '--by-cases',
+        action='store_true',
+        help="Step through benches by case.")
+    bench_parser.add_argument(
+        '--context',
+        type=lambda x: int(x, 0),
+        default=5,
+        help="Show this many lines of stdout on bench failure. "
+            "Defaults to 5.")
+    bench_parser.add_argument(
+        '--gdb',
+        action='store_true',
+        help="Drop into gdb on bench failure.")
+    bench_parser.add_argument(
+        '--gdb-case',
+        action='store_true',
+        help="Drop into gdb on bench failure but stop at the beginning "
+            "of the failing bench case.")
+    bench_parser.add_argument(
+        '--gdb-main',
+        action='store_true',
+        help="Drop into gdb on bench failure but stop at the beginning "
+            "of main.")
+    bench_parser.add_argument(
+        '--exec',
+        type=lambda e: e.split(),
+        help="Run under another executable.")
+    bench_parser.add_argument(
+        '--valgrind',
+        action='store_true',
+        help="Run under Valgrind to find memory errors. Implicitly sets "
+            "--isolate.")
+
+    # compilation flags
+    comp_parser = parser.add_argument_group('compilation options')
+    comp_parser.add_argument(
+        'bench_paths',
+        nargs='*',
+        help="Description of *.toml files to compile. May be a directory "
+            "or a list of paths.")
+    comp_parser.add_argument(
+        '-c', '--compile',
+        action='store_true',
+        help="Compile a bench suite or source file.")
+    comp_parser.add_argument(
+        '-s', '--source',
+        help="Source file to compile, possibly injecting internal benches.")
+    comp_parser.add_argument(
+        '--include',
+        default=HEADER_PATH,
+        help="Inject this header file into every compiled bench file. "
+            "Defaults to %r." % HEADER_PATH)
+    comp_parser.add_argument(
+        '-o', '--output',
+        help="Output file.")
+
+    # runner + bench_ids overlaps bench_paths, so we need to do some munging here
+    args = parser.parse_intermixed_args()
+    args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids
+    args.runner = args.runner or [RUNNER_PATH]
+
+    sys.exit(main(**{k: v
+        for k, v in vars(args).items()
+        if v is not None}))

scripts/summary.py

@@ -28,7 +28,8 @@ MERGES = {
     'add': (
         ['code_size', 'data_size', 'stack_frame', 'struct_size',
             'coverage_lines', 'coverage_branches',
-            'test_passed'],
+            'test_passed',
+            'bench_read', 'bench_prog', 'bench_erased'],
         lambda xs: sum(xs[1:], start=xs[0])
     ),
     'mul': (

scripts/tailpipe.py

@@ -48,7 +48,7 @@ def main(path='-', *, lines=1, sleep=0.01, keep_open=False):
             if not keep_open:
                 break
             # don't just flood open calls
-            time.sleep(sleep)
+            time.sleep(sleep or 0.1)
         done = True
 
     th.Thread(target=read, daemon=True).start()

scripts/test.py

@@ -388,7 +388,7 @@ def compile(test_paths, **args):
                         f.writeln('#define %-24s '
                             'TEST_IMPLICIT_DEFINE_COUNT+%d' % (define+'_i', i))
                         f.writeln('#define %-24s '
-                            'test_define(%s)' % (define, define+'_i'))
+                            'TEST_DEFINE(%s)' % (define, define+'_i'))
                         f.writeln('#endif')
                     f.writeln()
 
@@ -486,7 +486,7 @@ def compile(test_paths, **args):
                                         'TEST_IMPLICIT_DEFINE_COUNT+%d' % (
                                         define+'_i', i))
                                     f.writeln('#define %-24s '
-                                        'test_define(%s)' % (
+                                        'TEST_DEFINE(%s)' % (
                                         define, define+'_i'))
                                     f.writeln('#define '
                                         '__TEST__%s__NEEDS_UNDEF' % (
@@ -1018,7 +1018,9 @@ def run(runner, test_ids=[], **args):
         trace = openio(args['trace'], 'w', 1)
     output = None
     if args.get('output'):
-        output = TestOutput(args['output'], ['suite', 'case'], ['test_passed'])
+        output = TestOutput(args['output'],
+            ['suite', 'case'],
+            ['test_passed'])
 
     # measure runtime
     start = time.time()

scripts/tracebd.py

@@ -607,7 +607,7 @@ def main(path='-', *,
                 if not keep_open:
                     break
                 # don't just flood open calls
-                time.sleep(sleep)
+                time.sleep(sleep or 0.1)
         except KeyboardInterrupt:
             pass
     else:
@@ -627,7 +627,7 @@ def main(path='-', *,
                 if not keep_open:
                     break
                 # don't just flood open calls
-                time.sleep(sleep)
+                time.sleep(sleep or 0.1)
             done = True
 
         th.Thread(target=parse, daemon=True).start()

tests/test_alloc.toml

@@ -370,7 +370,7 @@ code = '''
 [cases.test_alloc_bad_blocks]
 in = "lfs.c"
 defines.ERASE_CYCLES = 0xffffffff
-defines.BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR'
+defines.BADBLOCK_BEHAVIOR = 'LFS_EMUBD_BADBLOCK_READERROR'
 code = '''
     lfs_t lfs;
     lfs_format(&lfs, cfg) => 0;
@@ -409,7 +409,7 @@ code = '''
 
     // but mark the head of our file as a "bad block", this is force our
     // scan to bail early
-    lfs_testbd_setwear(cfg, fileblock, 0xffffffff) => 0;
+    lfs_emubd_setwear(cfg, fileblock, 0xffffffff) => 0;
     lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
     strcpy((char*)buffer, "chomp");
     size = strlen("chomp");
@@ -424,7 +424,7 @@ code = '''
 
     // now reverse the "bad block" and try to write the file again until we
     // run out of space
-    lfs_testbd_setwear(cfg, fileblock, 0) => 0;
+    lfs_emubd_setwear(cfg, fileblock, 0) => 0;
     lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
     strcpy((char*)buffer, "chomp");
     size = strlen("chomp");

tests/test_badblocks.toml

@@ -6,18 +6,18 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 defines.NAMEMULT = 64
 defines.FILEMULT = 1
 code = '''
     for (lfs_block_t badblock = 2; badblock < BLOCK_COUNT; badblock++) {
-        lfs_testbd_setwear(cfg, badblock-1, 0) => 0;
-        lfs_testbd_setwear(cfg, badblock, 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, badblock-1, 0) => 0;
+        lfs_emubd_setwear(cfg, badblock, 0xffffffff) => 0;
 
         lfs_t lfs;
         lfs_format(&lfs, cfg) => 0;
@@ -86,17 +86,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 defines.NAMEMULT = 64
 defines.FILEMULT = 1
 code = '''
     for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) {
-        lfs_testbd_setwear(cfg, i+2, 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, i+2, 0xffffffff) => 0;
     }
 
     lfs_t lfs;
@@ -165,17 +165,17 @@ defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 defines.NAMEMULT = 64
 defines.FILEMULT = 1
 code = '''
     for (lfs_block_t i = 0; i < (BLOCK_COUNT-2)/2; i++) {
-        lfs_testbd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, (2*i) + 2, 0xffffffff) => 0;
     }
 
     lfs_t lfs;
@@ -244,15 +244,15 @@ code = '''
 defines.ERASE_CYCLES = 0xffffffff
 defines.ERASE_VALUE = [0x00, 0xff, -1]
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 code = '''
-    lfs_testbd_setwear(cfg, 0, 0xffffffff) => 0;
-    lfs_testbd_setwear(cfg, 1, 0xffffffff) => 0;
+    lfs_emubd_setwear(cfg, 0, 0xffffffff) => 0;
+    lfs_emubd_setwear(cfg, 1, 0xffffffff) => 0;
 
     lfs_t lfs;
     lfs_format(&lfs, cfg) => LFS_ERR_NOSPC;

tests/test_exhaustion.toml

@@ -4,11 +4,11 @@ defines.ERASE_CYCLES = 10
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 defines.FILES = 10
 code = '''
@@ -97,11 +97,11 @@ defines.ERASE_CYCLES = 10
 defines.BLOCK_COUNT = 256 # small bd so test runs faster
 defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2'
 defines.BADBLOCK_BEHAVIOR = [
-    'LFS_TESTBD_BADBLOCK_PROGERROR',
-    'LFS_TESTBD_BADBLOCK_ERASEERROR',
-    'LFS_TESTBD_BADBLOCK_READERROR',
-    'LFS_TESTBD_BADBLOCK_PROGNOOP',
-    'LFS_TESTBD_BADBLOCK_ERASENOOP',
+    'LFS_EMUBD_BADBLOCK_PROGERROR',
+    'LFS_EMUBD_BADBLOCK_ERASEERROR',
+    'LFS_EMUBD_BADBLOCK_READERROR',
+    'LFS_EMUBD_BADBLOCK_PROGNOOP',
+    'LFS_EMUBD_BADBLOCK_ERASENOOP',
 ]
 defines.FILES = 10
 code = '''
@@ -197,7 +197,7 @@ code = '''
 
     for (int run = 0; run < 2; run++) {
         for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) {
-            lfs_testbd_setwear(cfg, b,
+            lfs_emubd_setwear(cfg, b,
                     (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0;
         }
 
@@ -297,7 +297,7 @@ code = '''
 
     for (int run = 0; run < 2; run++) {
         for (lfs_block_t b = 0; b < BLOCK_COUNT; b++) {
-            lfs_testbd_setwear(cfg, b,
+            lfs_emubd_setwear(cfg, b,
                     (b < run_block_count[run]) ? 0 : ERASE_CYCLES) => 0;
         }
 
@@ -469,12 +469,12 @@ exhausted:
     LFS_WARN("completed %d cycles", cycle);
 
     // check the wear on our block device
-    lfs_testbd_wear_t minwear = -1;
-    lfs_testbd_wear_t totalwear = 0;
-    lfs_testbd_wear_t maxwear = 0;
+    lfs_emubd_wear_t minwear = -1;
+    lfs_emubd_wear_t totalwear = 0;
+    lfs_emubd_wear_t maxwear = 0;
     // skip 0 and 1 as superblock movement is intentionally avoided
     for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) {
-        lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b);
+        lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b);
         printf("%08x: wear %d\n", b, wear);
         assert(wear >= 0);
         if (wear < minwear) {
@@ -485,17 +485,17 @@ exhausted:
         }
         totalwear += wear;
     }
-    lfs_testbd_wear_t avgwear = totalwear / BLOCK_COUNT;
+    lfs_emubd_wear_t avgwear = totalwear / BLOCK_COUNT;
     LFS_WARN("max wear: %d cycles", maxwear);
     LFS_WARN("avg wear: %d cycles", totalwear / (int)BLOCK_COUNT);
     LFS_WARN("min wear: %d cycles", minwear);
 
     // find standard deviation^2
-    lfs_testbd_wear_t dev2 = 0;
+    lfs_emubd_wear_t dev2 = 0;
     for (lfs_block_t b = 2; b < BLOCK_COUNT; b++) {
-        lfs_testbd_wear_t wear = lfs_testbd_getwear(cfg, b);
+        lfs_emubd_wear_t wear = lfs_emubd_getwear(cfg, b);
         assert(wear >= 0);
-        lfs_testbd_swear_t diff = wear - avgwear;
+        lfs_emubd_swear_t diff = wear - avgwear;
         dev2 += diff*diff;
     }
     dev2 /= totalwear;

tests/test_move.toml

@@ -1638,15 +1638,15 @@ code = '''
     if (RELOCATIONS & 0x1) {
         lfs_dir_t dir;
         lfs_dir_open(&lfs, &dir, "/parent");
-        lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
-        lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
         lfs_dir_close(&lfs, &dir) => 0;
     }
     if (RELOCATIONS & 0x2) {
         lfs_dir_t dir;
         lfs_dir_open(&lfs, &dir, "/parent/child");
-        lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
-        lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
         lfs_dir_close(&lfs, &dir) => 0;
     }
 
@@ -1784,22 +1784,22 @@ code = '''
     if (RELOCATIONS & 0x1) {
         lfs_dir_t dir;
         lfs_dir_open(&lfs, &dir, "/parent");
-        lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
-        lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
         lfs_dir_close(&lfs, &dir) => 0;
     }
     if (RELOCATIONS & 0x2) {
         lfs_dir_t dir;
         lfs_dir_open(&lfs, &dir, "/parent/sibling");
-        lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
-        lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
         lfs_dir_close(&lfs, &dir) => 0;
     }
     if (RELOCATIONS & 0x4) {
         lfs_dir_t dir;
         lfs_dir_open(&lfs, &dir, "/parent/child");
-        lfs_testbd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
-        lfs_testbd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[0], 0xffffffff) => 0;
+        lfs_emubd_setwear(cfg, dir.m.pair[1], 0xffffffff) => 0;
         lfs_dir_close(&lfs, &dir) => 0;
     }