From 0b3261e1aa1356f291030622a91dca244324cd0b Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:40:55 +0100 Subject: [PATCH] Bound native fixture controls independently The NAS suite exhausts its aggregate alarm after fifteen successful controls on macOS Intel. Run each discovered NAS and updater control under its existing bound, retaining the native phase's ten-minute alarm and all installer operation deadlines. Keep required controls mandatory and reject skipped, absent or failed execution. Exercise discovery and false-pass rejection with real Python controls, including a newly added method. --- scripts/installtests/install_sh_test.go | 38 +----- .../installtests/pulse_auto_update_test.go | 48 +------ .../python_fixture_controls_test.go | 123 ++++++++++++++++++ 3 files changed, 131 insertions(+), 78 deletions(-) create mode 100644 scripts/installtests/python_fixture_controls_test.go diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go index 859b369e3c..ff6d33bd77 100644 --- a/scripts/installtests/install_sh_test.go +++ b/scripts/installtests/install_sh_test.go @@ -8599,14 +8599,7 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T) if runtime.GOOS == "windows" { t.Skip("TrueNAS SCALE shell controls; Windows has its own native SCM gate") } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_truenas_teardown.py")) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("TrueNAS SCALE boot-task ownership/retirement controls: %v\n%s", err, output) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_truenas_teardown.py"), "AgentTrueNASTeardown", []string{ "test_unavailable_or_untrusted_task_inventory_cannot_remove_recovery", "test_foreign_or_indirect_bootstrap_is_not_mutated", "test_missing_activation_target_cannot_hide_an_owned_task", @@ -8623,29 +8616,14 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T) "test_disabled_absent_missing_bootstrap_and_repeated_removal_are_observed", "test_ordinary_linux_state_is_not_appliance_evidence", "test_process_read_exposes_only_roles_and_real_timeout_child_is_stopped", - } { - if !strings.Contains(string(output), name+" (__main__.AgentTrueNASTeardown."+name+") ... ok") { - t.Fatalf("TrueNAS SCALE control %s did not pass:\n%s", name, output) - } - } - if !strings.Contains(string(output), "\nRan 16 tests in ") || !strings.Contains(string(output), "\nOK\n") { - t.Fatalf("incomplete TrueNAS SCALE verdict:\n%s", output) - } - t.Logf("%s", output) + }, 2*time.Minute) } func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix NAS shell controls; Windows retains its separate native SCM gate") } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_nas_teardown.py")) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("NAS boot-owner/watchdog production caller controls: %v\n%s", err, output) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_nas_teardown.py"), "AgentNASTeardown", []string{ "test_complete_process_evidence_is_required_before_boot_or_remote_mutation", "test_unknown_relative_foreign_or_deferred_boot_owners_are_not_signalled", "test_non_regular_foreign_wrapper_and_unselected_state_preserve_recovery", @@ -8669,13 +8647,5 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) { "test_private_process_reader_requests_native_basename_and_full_argv", "test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats", "test_caller_has_no_blanket_nas_or_legacy_tree_deletion", - } { - if !strings.Contains(string(output), name+" (__main__.AgentNASTeardown."+name+") ... ok") { - t.Fatalf("NAS removal control %s did not pass:\n%s", name, output) - } - } - if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") { - t.Fatalf("incomplete NAS removal verdict:\n%s", output) - } - t.Logf("%s", output) + }, 2*time.Minute) } diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go index 63a0f401bf..a147c00f44 100644 --- a/scripts/installtests/pulse_auto_update_test.go +++ b/scripts/installtests/pulse_auto_update_test.go @@ -162,18 +162,7 @@ func TestAutoUpdateDownloadFailures(t *testing.T) { // Service-state admission is independent of the signed/metadata transaction // matrix below; no native service or published input is used by this fixture. func TestAutoUpdateServiceStateFailsClosed(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateServiceState") - cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out) - } - // Keep the manager/collector controls part of actual package acceptance, not just - // an optional Python check. A green fixture with these methods removed - // would otherwise silently lose early-failure and final-line coverage. - required := []string{ + runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateServiceState", []string{ "test_manager_diagnostics_refuse_initial_update_and_restart", "test_manager_diagnostics_after_install_retain_backup_without_mutation", "test_manager_diagnostics_after_stop_retain_backup_without_restart", @@ -181,46 +170,17 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) { "test_rollback_keeps_backup_until_restored_running_executable_matches", "test_early_collector_failure_survives_later_success_and_drains_installer", "test_unterminated_final_installer_line_is_collected_and_its_failure_retained", - } - for _, name := range required { - passed := false - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") { - passed = true - } - } - if !passed { - t.Fatalf("required updater collector control %s did not pass:\n%s", name, out) - } - } + }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) } // Keep the full real-sink matrix mandatory, but give it its own existing-size // bound rather than exhausting the service-state suite's manager deadlines. func TestAutoUpdateLogSinkFailures(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateLogSinks") - cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("unattended real-log sink controls: %v\n%s", err, out) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateLogSinks", []string{ "test_logger_double_keeps_complete_argument_frames_and_exit_sequence", "test_real_log_retains_each_sink_failure_in_conditional_calls", "test_real_logger_failure_drains_installer_and_preserves_recovery", - } { - passed := false - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") { - passed = true - } - } - if !passed { - t.Fatalf("required real-log sink control %s did not pass:\n%s", name, out) - } - } + }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) } // A real owned process retains its old executable after its disk path changes. diff --git a/scripts/installtests/python_fixture_controls_test.go b/scripts/installtests/python_fixture_controls_test.go new file mode 100644 index 0000000000..e24a56321b --- /dev/null +++ b/scripts/installtests/python_fixture_controls_test.go @@ -0,0 +1,123 @@ +package installtests + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// A suite's cumulative runtime is not a service or installer deadline. Give +// each independent control the existing bound, while the native phase keeps +// its overall Go timeout. Discovery runs every method, including future ones, +// and the required list prevents deleting a safety control unnoticed. +func runPythonFixtureControls(t *testing.T, script, class string, required []string, bound time.Duration, env ...string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), bound) + controls, err := discoverPythonFixtureControls(ctx, script, class, required, env...) + cancel() + if err != nil { + t.Fatal(err) + } + t.Logf("Discovered %d %s controls; each retains its %s bound", len(controls), class, bound) + for _, name := range controls { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + output, err := executePythonFixtureControl(ctx, script, class, name, env...) + if err != nil { + t.Fatal(err) + } + t.Logf("%s", output) + }) + } +} + +func discoverPythonFixtureControls(ctx context.Context, script, class string, required []string, env ...string) ([]string, error) { + const discover = `import json, runpy, sys, unittest +namespace = runpy.run_path(sys.argv[1]) +suite = unittest.defaultTestLoader.loadTestsFromTestCase(namespace[sys.argv[2]]) +print(json.dumps([test._testMethodName for test in suite])) +` + cmd := exec.CommandContext(ctx, "python3", "-I", "-S", "-c", discover, script, class) + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("discover %s controls: %w (context %v)\n%s", class, err, ctx.Err(), output) + } + var controls []string + if err := json.Unmarshal(output, &controls); err != nil { + return nil, fmt.Errorf("decode %s controls: %w\n%s", class, err, output) + } + if len(controls) == 0 { + return nil, fmt.Errorf("no controls discovered in %s", class) + } + for _, name := range required { + found := false + for _, control := range controls { + found = found || control == name + } + if !found { + return nil, fmt.Errorf("required %s control %s is absent from discovery", class, name) + } + } + return controls, nil +} + +func executePythonFixtureControl(ctx context.Context, script, class, name string, env ...string) ([]byte, error) { + selector := class + "." + name + cmd := exec.CommandContext(ctx, "python3", "-I", "-S", script, "-v", selector) + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + if err != nil { + return output, fmt.Errorf("%s: %w (context %v)\n%s", selector, err, ctx.Err(), output) + } + if !strings.Contains(string(output), name+" (__main__."+selector+") ... ok") || + !strings.Contains(string(output), "\nRan 1 test in ") || + !strings.Contains(string(output), "\nOK\n") { + return output, fmt.Errorf("%s did not execute exactly one passing control:\n%s", selector, output) + } + return output, nil +} + +func TestPythonFixtureControlsRetainDiscoveryAndRejectFalsePasses(t *testing.T) { + script := filepath.Join(t.TempDir(), "controls.py") + const source = `import unittest +class Controls(unittest.TestCase): + def test_added(self): pass + def test_fail(self): self.fail("retained adverse control") + def test_pass(self): pass + @unittest.skip("must not be credited") + def test_skip(self): pass +if __name__ == "__main__": unittest.main(verbosity=2) +` + if err := os.WriteFile(script, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + controls, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_pass"}) + if err != nil { + t.Fatal(err) + } + if want := []string{"test_added", "test_fail", "test_pass", "test_skip"}; !reflect.DeepEqual(controls, want) { + t.Fatalf("new controls must be discovered too: got %v, want %v", controls, want) + } + if _, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_removed"}); err == nil { + t.Fatal("a deleted required control must fail discovery") + } + for _, name := range []string{"test_pass", "test_skip", "test_fail", "test_missing"} { + t.Run(name, func(t *testing.T) { + _, err := executePythonFixtureControl(ctx, script, "Controls", name) + if (err == nil) != (name == "test_pass") { + t.Fatalf("execution verdict for %s: %v", name, err) + } + }) + } +}