Bound native fixture controls independently

The NAS suite exhausts its aggregate alarm after fifteen successful
controls on macOS Intel. Run each discovered NAS and updater control
under its existing bound, retaining the native phase's ten-minute alarm
and all installer operation deadlines.

Keep required controls mandatory and reject skipped, absent or failed
execution. Exercise discovery and false-pass rejection with real Python
controls, including a newly added method.

scripts/installtests/install_sh_test.go

@@ -8599,14 +8599,7 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T)
 	if runtime.GOOS == "windows" {
 		t.Skip("TrueNAS SCALE shell controls; Windows has its own native SCM gate")
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_truenas_teardown.py"))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("TrueNAS SCALE boot-task ownership/retirement controls: %v\n%s", err, output)
-	}
-	for _, name := range []string{
+	runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_truenas_teardown.py"), "AgentTrueNASTeardown", []string{
 		"test_unavailable_or_untrusted_task_inventory_cannot_remove_recovery",
 		"test_foreign_or_indirect_bootstrap_is_not_mutated",
 		"test_missing_activation_target_cannot_hide_an_owned_task",
@@ -8623,29 +8616,14 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T)
 		"test_disabled_absent_missing_bootstrap_and_repeated_removal_are_observed",
 		"test_ordinary_linux_state_is_not_appliance_evidence",
 		"test_process_read_exposes_only_roles_and_real_timeout_child_is_stopped",
-	} {
-		if !strings.Contains(string(output), name+" (__main__.AgentTrueNASTeardown."+name+") ... ok") {
-			t.Fatalf("TrueNAS SCALE control %s did not pass:\n%s", name, output)
-		}
-	}
-	if !strings.Contains(string(output), "\nRan 16 tests in ") || !strings.Contains(string(output), "\nOK\n") {
-		t.Fatalf("incomplete TrueNAS SCALE verdict:\n%s", output)
-	}
-	t.Logf("%s", output)
+	}, 2*time.Minute)
 }
 
 func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("Unix NAS shell controls; Windows retains its separate native SCM gate")
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_nas_teardown.py"))
-	output, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("NAS boot-owner/watchdog production caller controls: %v\n%s", err, output)
-	}
-	for _, name := range []string{
+	runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_nas_teardown.py"), "AgentNASTeardown", []string{
 		"test_complete_process_evidence_is_required_before_boot_or_remote_mutation",
 		"test_unknown_relative_foreign_or_deferred_boot_owners_are_not_signalled",
 		"test_non_regular_foreign_wrapper_and_unselected_state_preserve_recovery",
@@ -8669,13 +8647,5 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) {
 		"test_private_process_reader_requests_native_basename_and_full_argv",
 		"test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats",
 		"test_caller_has_no_blanket_nas_or_legacy_tree_deletion",
-	} {
-		if !strings.Contains(string(output), name+" (__main__.AgentNASTeardown."+name+") ... ok") {
-			t.Fatalf("NAS removal control %s did not pass:\n%s", name, output)
-		}
-	}
-	if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") {
-		t.Fatalf("incomplete NAS removal verdict:\n%s", output)
-	}
-	t.Logf("%s", output)
+	}, 2*time.Minute)
 }

scripts/installtests/pulse_auto_update_test.go

@@ -162,18 +162,7 @@ func TestAutoUpdateDownloadFailures(t *testing.T) {
 // Service-state admission is independent of the signed/metadata transaction
 // matrix below; no native service or published input is used by this fixture.
 func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateServiceState")
-	cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out)
-	}
-	// Keep the manager/collector controls part of actual package acceptance, not just
-	// an optional Python check. A green fixture with these methods removed
-	// would otherwise silently lose early-failure and final-line coverage.
-	required := []string{
+	runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateServiceState", []string{
 		"test_manager_diagnostics_refuse_initial_update_and_restart",
 		"test_manager_diagnostics_after_install_retain_backup_without_mutation",
 		"test_manager_diagnostics_after_stop_retain_backup_without_restart",
@@ -181,46 +170,17 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) {
 		"test_rollback_keeps_backup_until_restored_running_executable_matches",
 		"test_early_collector_failure_survives_later_success_and_drains_installer",
 		"test_unterminated_final_installer_line_is_collected_and_its_failure_retained",
-	}
-	for _, name := range required {
-		passed := false
-		for _, line := range strings.Split(string(out), "\n") {
-			if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {
-				passed = true
-			}
-		}
-		if !passed {
-			t.Fatalf("required updater collector control %s did not pass:\n%s", name, out)
-		}
-	}
+	}, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
 }
 
 // Keep the full real-sink matrix mandatory, but give it its own existing-size
 // bound rather than exhausting the service-state suite's manager deadlines.
 func TestAutoUpdateLogSinkFailures(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateLogSinks")
-	cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("unattended real-log sink controls: %v\n%s", err, out)
-	}
-	for _, name := range []string{
+	runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateLogSinks", []string{
 		"test_logger_double_keeps_complete_argument_frames_and_exit_sequence",
 		"test_real_log_retains_each_sink_failure_in_conditional_calls",
 		"test_real_logger_failure_drains_installer_and_preserves_recovery",
-	} {
-		passed := false
-		for _, line := range strings.Split(string(out), "\n") {
-			if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") {
-				passed = true
-			}
-		}
-		if !passed {
-			t.Fatalf("required real-log sink control %s did not pass:\n%s", name, out)
-		}
-	}
+	}, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh"))
 }
 
 // A real owned process retains its old executable after its disk path changes.

scripts/installtests/python_fixture_controls_test.go

@@ -0,0 +1,123 @@
+package installtests
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+)
+
+// A suite's cumulative runtime is not a service or installer deadline. Give
+// each independent control the existing bound, while the native phase keeps
+// its overall Go timeout. Discovery runs every method, including future ones,
+// and the required list prevents deleting a safety control unnoticed.
+func runPythonFixtureControls(t *testing.T, script, class string, required []string, bound time.Duration, env ...string) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), bound)
+	controls, err := discoverPythonFixtureControls(ctx, script, class, required, env...)
+	cancel()
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("Discovered %d %s controls; each retains its %s bound", len(controls), class, bound)
+	for _, name := range controls {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), bound)
+			defer cancel()
+			output, err := executePythonFixtureControl(ctx, script, class, name, env...)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%s", output)
+		})
+	}
+}
+
+func discoverPythonFixtureControls(ctx context.Context, script, class string, required []string, env ...string) ([]string, error) {
+	const discover = `import json, runpy, sys, unittest
+namespace = runpy.run_path(sys.argv[1])
+suite = unittest.defaultTestLoader.loadTestsFromTestCase(namespace[sys.argv[2]])
+print(json.dumps([test._testMethodName for test in suite]))
+`
+	cmd := exec.CommandContext(ctx, "python3", "-I", "-S", "-c", discover, script, class)
+	cmd.Env = append(os.Environ(), env...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("discover %s controls: %w (context %v)\n%s", class, err, ctx.Err(), output)
+	}
+	var controls []string
+	if err := json.Unmarshal(output, &controls); err != nil {
+		return nil, fmt.Errorf("decode %s controls: %w\n%s", class, err, output)
+	}
+	if len(controls) == 0 {
+		return nil, fmt.Errorf("no controls discovered in %s", class)
+	}
+	for _, name := range required {
+		found := false
+		for _, control := range controls {
+			found = found || control == name
+		}
+		if !found {
+			return nil, fmt.Errorf("required %s control %s is absent from discovery", class, name)
+		}
+	}
+	return controls, nil
+}
+
+func executePythonFixtureControl(ctx context.Context, script, class, name string, env ...string) ([]byte, error) {
+	selector := class + "." + name
+	cmd := exec.CommandContext(ctx, "python3", "-I", "-S", script, "-v", selector)
+	cmd.Env = append(os.Environ(), env...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return output, fmt.Errorf("%s: %w (context %v)\n%s", selector, err, ctx.Err(), output)
+	}
+	if !strings.Contains(string(output), name+" (__main__."+selector+") ... ok") ||
+		!strings.Contains(string(output), "\nRan 1 test in ") ||
+		!strings.Contains(string(output), "\nOK\n") {
+		return output, fmt.Errorf("%s did not execute exactly one passing control:\n%s", selector, output)
+	}
+	return output, nil
+}
+
+func TestPythonFixtureControlsRetainDiscoveryAndRejectFalsePasses(t *testing.T) {
+	script := filepath.Join(t.TempDir(), "controls.py")
+	const source = `import unittest
+class Controls(unittest.TestCase):
+    def test_added(self): pass
+    def test_fail(self): self.fail("retained adverse control")
+    def test_pass(self): pass
+    @unittest.skip("must not be credited")
+    def test_skip(self): pass
+if __name__ == "__main__": unittest.main(verbosity=2)
+`
+	if err := os.WriteFile(script, []byte(source), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	controls, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_pass"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := []string{"test_added", "test_fail", "test_pass", "test_skip"}; !reflect.DeepEqual(controls, want) {
+		t.Fatalf("new controls must be discovered too: got %v, want %v", controls, want)
+	}
+	if _, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_removed"}); err == nil {
+		t.Fatal("a deleted required control must fail discovery")
+	}
+	for _, name := range []string{"test_pass", "test_skip", "test_fail", "test_missing"} {
+		t.Run(name, func(t *testing.T) {
+			_, err := executePythonFixtureControl(ctx, script, "Controls", name)
+			if (err == nil) != (name == "test_pass") {
+				t.Fatalf("execution verdict for %s: %v", name, err)
+			}
+		})
+	}
+}