Parallelize src/test/test into chunks.

First, we introduce a flag to teach src/test/test to split its work
into chunks.  Then we replace our invocation of src/test/test in our
"make check" target with a set of 8 scripts that invoke the first
8th of the tests, the second 8th, and so on.

This change makes our "make -kj4 check" target in our hardened
gitlab build more than twice as fast, since src/test/test was taking
the longest to finish.

Closes 40098.

11 files changed, 89 insertions, 3 deletions

diff --git a/changes/parallel_unit_test b/changes/parallel_unit_test
new file mode 100644
index 0000000000..79de28636d
--- /dev/null
+++ b/changes/parallel_unit_test
@@ -0,0 +1,4 @@
+  o Minor features (tests):
+    - Our "make check" target now runs the unit tests in 8 parallel chunks.
+      Doing this speeds up hardened CI builds by more than a factor of two.
+      Closes ticket 40098.
diff --git a/src/test/include.am b/src/test/include.am
index ecb7689579..75861fb9ef 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -23,7 +23,15 @@ TESTSCRIPTS = \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
-	src/test/test_switch_id.sh
+	src/test/test_switch_id.sh \
+        src/test/unittest_part1.sh \
+        src/test/unittest_part2.sh \
+        src/test/unittest_part3.sh \
+        src/test/unittest_part4.sh \
+        src/test/unittest_part5.sh \
+        src/test/unittest_part6.sh \
+        src/test/unittest_part7.sh \
+        src/test/unittest_part8.sh
 
 if USE_RUST
 TESTSCRIPTS += \
@@ -35,7 +43,7 @@ TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.s
 TESTSCRIPTS += src/test/test_rebind.sh
 endif
 
-TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
+TESTS += src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
 	src/test/test_keygen.sh \
 	src/test/test_key_expiration.sh \
@@ -369,7 +377,15 @@ EXTRA_DIST += \
 	src/test/test_workqueue_efd2.sh \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
-	src/test/test_workqueue_socketpair.sh
+	src/test/test_workqueue_socketpair.sh \
+        src/test/unittest_part1.sh \
+        src/test/unittest_part2.sh \
+        src/test/unittest_part3.sh \
+        src/test/unittest_part4.sh \
+        src/test/unittest_part5.sh \
+        src/test/unittest_part6.sh \
+        src/test/unittest_part7.sh \
+        src/test/unittest_part8.sh
 
 test-rust:
 	$(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh"
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 2c9c4538b9..daa7aa524a 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -278,6 +278,8 @@ main(int c, const char **v)
   control_initialize_event_queue();
   configure_backtrace_handler(get_version());
 
+  unsigned num=1, den=1;
+
   for (i_out = i = 1; i < c; ++i) {
     if (!strcmp(v[i], "--warn")) {
       loglevel = LOG_WARN;
@@ -289,6 +291,19 @@ main(int c, const char **v)
       loglevel = LOG_DEBUG;
     } else if (!strcmp(v[i], "--accel")) {
       accel_crypto = 1;
+    } else if (!strcmp(v[i], "--fraction")) {
+      if (i+1 == c) {
+        printf("--fraction needs an argument.\n");
+        return 1;
+      }
+      const char *fracstr = v[++i];
+      char ch;
+      if (sscanf(fracstr, "%u/%u%c", &num, &den, &ch) != 2) {
+        printf("--fraction expects a fraction as an input.\n");
+      }
+      if (den == 0 || num == 0 || num > den) {
+        printf("--fraction expects a valid fraction as an input.\n");
+      }
     } else {
       v[i_out++] = v[i];
     }
@@ -363,6 +378,33 @@ main(int c, const char **v)
     smartlist_free(skip);
   }
 
+  if (den != 1) {
+    // count the tests. Linear but fast.
+    unsigned n_tests = 0;
+    struct testgroup_t *tg;
+    struct testcase_t *tc;
+    for (tg = testgroups; tg->prefix != NULL; ++tg) {
+      for (tc = tg->cases; tc->name != NULL; ++tc) {
+        ++n_tests;
+      }
+    }
+    // Which tests should we run?  This can give iffy results if den is huge
+    // but it doesn't actually matter in practice.
+    unsigned tests_per_chunk = CEIL_DIV(n_tests, den);
+    unsigned start_at = (num-1) * tests_per_chunk;
+
+    // Skip the tests that are outside of the range.
+    unsigned idx = 0;
+    for (tg = testgroups; tg->prefix != NULL; ++tg) {
+      for (tc = tg->cases; tc->name != NULL; ++tc) {
+        if (idx < start_at || idx >= start_at + tests_per_chunk) {
+          tc->flags |= TT_SKIP;
+        }
+        ++idx;
+      }
+    }
+  }
+
   int have_failed = (tinytest_main(c, v, testgroups) != 0);
 
   free_pregenerated_keys();
diff --git a/src/test/unittest_part1.sh b/src/test/unittest_part1.sh
new file mode 100755
index 0000000000..5be0f499f9
--- /dev/null
+++ b/src/test/unittest_part1.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 1/8
diff --git a/src/test/unittest_part2.sh b/src/test/unittest_part2.sh
new file mode 100755
index 0000000000..9a614eb8c1
--- /dev/null
+++ b/src/test/unittest_part2.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 2/8
diff --git a/src/test/unittest_part3.sh b/src/test/unittest_part3.sh
new file mode 100755
index 0000000000..5cbc3fe495
--- /dev/null
+++ b/src/test/unittest_part3.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 3/8
diff --git a/src/test/unittest_part4.sh b/src/test/unittest_part4.sh
new file mode 100755
index 0000000000..bc6fe01f68
--- /dev/null
+++ b/src/test/unittest_part4.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 4/8
diff --git a/src/test/unittest_part5.sh b/src/test/unittest_part5.sh
new file mode 100755
index 0000000000..9bbff34fb8
--- /dev/null
+++ b/src/test/unittest_part5.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 5/8
diff --git a/src/test/unittest_part6.sh b/src/test/unittest_part6.sh
new file mode 100755
index 0000000000..2d5eaa8a28
--- /dev/null
+++ b/src/test/unittest_part6.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 6/8
diff --git a/src/test/unittest_part7.sh b/src/test/unittest_part7.sh
new file mode 100755
index 0000000000..5e6ce2aea5
--- /dev/null
+++ b/src/test/unittest_part7.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 7/8
diff --git a/src/test/unittest_part8.sh b/src/test/unittest_part8.sh
new file mode 100755
index 0000000000..7fea9c9c7f
--- /dev/null
+++ b/src/test/unittest_part8.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+"${abs_top_builddir:-.}/src/test/test" --fraction 8/8