test: Add parallel test execution support

Running the full test suite takes a long time. It would be useful to
distribute tests across multiple sandbox instances to speed up testing.

Add support for running tests in parallel across multiple sandbox
instances. Each worker runs a subset of tests based on its worker ID.

Add -P<n>:<w> option to the ut command where n is the total number of
workers and w is this worker's ID (0 to n-1). Tests are distributed
by index modulo number of workers.

Series-to: concept
Series-cc: heinrich
Cover-letter:
test: Various improvements to unit-test infrastructure

This series adds several improvements to the unit-test infrastructure:

- Move disk images to the persistent-data directory so they don't
  pollute the source tree
- Add a way to keep pytest-created artefacts for faster iteration on
  C tests
- Add a helper to simplify running manual unit tests from Python
- Allow combined flags with the ut command (e.g. -Efm)
- Add a -E flag to emit machine-readable result lines
- Add a -P flag to distribute tests across parallel workers
- Add -m as an alias for -f (force manual tests)

These changes make it easier to run and debug tests, especially when
iterating on C test code.
END

Co-developed-by: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Simon Glass <simon.glass@canonical.com>
Series-links: 1:94

doc/Makefile

@@ -6,7 +6,7 @@ subdir-y :=
 
 # You can set these variables from the command line.
 SPHINXBUILD   = sphinx-build
-SPHINXOPTS    = -q -W
+SPHINXOPTS    = -W
 SPHINXDIRS    = .
 _SPHINXDIRS   = $(patsubst $(srctree)/doc/%/conf.py,%,$(wildcard $(srctree)/doc/*/conf.py))
 SPHINX_CONF   = conf.py
@@ -51,8 +51,8 @@ loop_cmd = $(echo-cmd) $(cmd_$(1)) || exit;
 # $5 reST source folder relative to $(srctree)/$(src),
 #    e.g. "media" for the linux-tv book-set at ./doc/media
 
-quiet_cmd_sphinx =
+quiet_cmd_sphinx = SPHINX  $@ --> file://$(abspath $(BUILDDIR)/$3/$4)
-      cmd_sphinx = $(MAKE) -s BUILDDIR=$(abspath $(BUILDDIR)) $(build)=doc/media $2 && \
+      cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=doc/media $2 && \
 	PYTHONDONTWRITEBYTECODE=1 \
 	BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \
 	$(SPHINXBUILD) \

doc/conf.py

@@ -377,6 +377,7 @@ latex_elements = {
 
 cjk_cmd = check_output(['fc-list', '--format="%{family[0]}\n"']).decode('utf-8', 'ignore')
 if cjk_cmd.find("Noto Sans CJK SC") >= 0:
+    print ("enabling CJK for LaTeX builder")
     latex_elements['preamble']  += '''
 	% This is needed for translations
         \\usepackage{xeCJK}

doc/media/Makefile

@@ -12,7 +12,8 @@ gen_rst = \
 	echo ${PARSER} $< $@ $(SRC_DIR)/$(notdir $@).exceptions; \
 	${PARSER} $< $@ $(SRC_DIR)/$(notdir $@).exceptions
 
-quiet_gen_rst = ${PARSER} $< $@ $(SRC_DIR)/$(notdir $@).exceptions
+quiet_gen_rst = echo '  PARSE   $(patsubst $(srctree)/%,%,$<)'; \
+	${PARSER} $< $@ $(SRC_DIR)/$(notdir $@).exceptions
 
 silent_gen_rst = ${gen_rst}
 

test/py/tests/test_fs/test_ext4l.py

@@ -72,49 +72,69 @@ class TestExt4l:
     def test_probe(self, ubman, ext4_image):
         """Test that ext4l can probe an ext4 filesystem."""
         with ubman.log.section('Test ext4l probe'):
-            ubman.run_ut('fs', 'fs_test_ext4l_probe', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_probe_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_msgs(self, ubman, ext4_image):
         """Test that ext4l_msgs env var produces mount messages."""
         with ubman.log.section('Test ext4l msgs'):
-            ubman.run_ut('fs', 'fs_test_ext4l_msgs', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_msgs_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_ls(self, ubman, ext4_image):
         """Test that ext4l can list directory contents."""
         with ubman.log.section('Test ext4l ls'):
-            ubman.run_ut('fs', 'fs_test_ext4l_ls', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_ls_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_opendir(self, ubman, ext4_image):
         """Test that ext4l can iterate directory entries."""
         with ubman.log.section('Test ext4l opendir'):
-            ubman.run_ut('fs', 'fs_test_ext4l_opendir', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_opendir_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_exists(self, ubman, ext4_image):
         """Test that ext4l_exists reports file existence correctly."""
         with ubman.log.section('Test ext4l exists'):
-            ubman.run_ut('fs', 'fs_test_ext4l_exists', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_exists_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_size(self, ubman, ext4_image):
         """Test that ext4l_size reports file size correctly."""
         with ubman.log.section('Test ext4l size'):
-            ubman.run_ut('fs', 'fs_test_ext4l_size', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_size_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_read(self, ubman, ext4_image):
         """Test that ext4l can read file contents."""
         with ubman.log.section('Test ext4l read'):
-            ubman.run_ut('fs', 'fs_test_ext4l_read', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_read_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_uuid(self, ubman, ext4_image):
         """Test that ext4l can return the filesystem UUID."""
         with ubman.log.section('Test ext4l uuid'):
-            ubman.run_ut('fs', 'fs_test_ext4l_uuid', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_uuid_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_statfs(self, ubman, ext4_image):
         """Test that ext4l can return filesystem statistics."""
         with ubman.log.section('Test ext4l statfs'):
-            ubman.run_ut('fs', 'fs_test_ext4l_statfs', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_statfs_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output
 
     def test_fsinfo(self, ubman, ext4_image):
         """Test that fsinfo command displays filesystem statistics."""
         with ubman.log.section('Test ext4l fsinfo'):
-            ubman.run_ut('fs', 'fs_test_ext4l_fsinfo', fs_image=ext4_image)
+            output = ubman.run_command(
+                f'ut -f fs fs_test_ext4l_fsinfo_norun fs_image={ext4_image}')
+            assert 'failures: 0' in output

test/py/tests/test_vbe.py

@@ -87,6 +87,7 @@ bootm loados
 bootm prep
 fdt addr
 fdt print
+ut -f bootstd vbe_test_fixup_norun
 '''
 
 @pytest.mark.boardspec('sandbox')
@@ -116,9 +117,9 @@ def test_vbe_os_request(ubman):
     cmd = base_script % params
 
     with ubman.log.section('Kernel load'):
-        ubman.run_command_list(cmd.splitlines())
+        output = ubman.run_command_list(cmd.splitlines())
 
-    ubman.run_ut('bootstd', 'vbe_test_fixup')
+    assert 'failures: 0' in output[-1]
 
 @pytest.mark.boardspec('sandbox')
 def test_vbe_extlinux_fit_no_oem(ubman):