diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go index 859b369e3c..ff6d33bd77 100644 --- a/scripts/installtests/install_sh_test.go +++ b/scripts/installtests/install_sh_test.go @@ -8599,14 +8599,7 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T) if runtime.GOOS == "windows" { t.Skip("TrueNAS SCALE shell controls; Windows has its own native SCM gate") } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_truenas_teardown.py")) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("TrueNAS SCALE boot-task ownership/retirement controls: %v\n%s", err, output) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_truenas_teardown.py"), "AgentTrueNASTeardown", []string{ "test_unavailable_or_untrusted_task_inventory_cannot_remove_recovery", "test_foreign_or_indirect_bootstrap_is_not_mutated", "test_missing_activation_target_cannot_hide_an_owned_task", @@ -8623,29 +8616,14 @@ func TestInstallSHTrueNASScaleBootTasksRetireBeforeRecoveryErasure(t *testing.T) "test_disabled_absent_missing_bootstrap_and_repeated_removal_are_observed", "test_ordinary_linux_state_is_not_appliance_evidence", "test_process_read_exposes_only_roles_and_real_timeout_child_is_stopped", - } { - if !strings.Contains(string(output), name+" (__main__.AgentTrueNASTeardown."+name+") ... ok") { - t.Fatalf("TrueNAS SCALE control %s did not pass:\n%s", name, output) - } - } - if !strings.Contains(string(output), "\nRan 16 tests in ") || !strings.Contains(string(output), "\nOK\n") { - t.Fatalf("incomplete TrueNAS SCALE verdict:\n%s", output) - } - t.Logf("%s", output) + }, 2*time.Minute) } func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Unix NAS shell controls; Windows retains its separate native SCM gate") } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "installtests", "agent_nas_teardown.py")) - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("NAS boot-owner/watchdog production caller controls: %v\n%s", err, output) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "installtests", "agent_nas_teardown.py"), "AgentNASTeardown", []string{ "test_complete_process_evidence_is_required_before_boot_or_remote_mutation", "test_unknown_relative_foreign_or_deferred_boot_owners_are_not_signalled", "test_non_regular_foreign_wrapper_and_unselected_state_preserve_recovery", @@ -8669,13 +8647,5 @@ func TestInstallSHNASBootOwnersQuiesceBeforeRecoveryErasure(t *testing.T) { "test_private_process_reader_requests_native_basename_and_full_argv", "test_generated_collector_observation_keeps_exact_identity_on_both_unix_formats", "test_caller_has_no_blanket_nas_or_legacy_tree_deletion", - } { - if !strings.Contains(string(output), name+" (__main__.AgentNASTeardown."+name+") ... ok") { - t.Fatalf("NAS removal control %s did not pass:\n%s", name, output) - } - } - if !strings.Contains(string(output), "\nRan 23 tests in ") || !strings.Contains(string(output), "\nOK\n") { - t.Fatalf("incomplete NAS removal verdict:\n%s", output) - } - t.Logf("%s", output) + }, 2*time.Minute) } diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go index 63a0f401bf..a147c00f44 100644 --- a/scripts/installtests/pulse_auto_update_test.go +++ b/scripts/installtests/pulse_auto_update_test.go @@ -162,18 +162,7 @@ func TestAutoUpdateDownloadFailures(t *testing.T) { // Service-state admission is independent of the signed/metadata transaction // matrix below; no native service or published input is used by this fixture. func TestAutoUpdateServiceStateFailsClosed(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateServiceState") - cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("unattended service state/rollback controls: %v\n%s", err, out) - } - // Keep the manager/collector controls part of actual package acceptance, not just - // an optional Python check. A green fixture with these methods removed - // would otherwise silently lose early-failure and final-line coverage. - required := []string{ + runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateServiceState", []string{ "test_manager_diagnostics_refuse_initial_update_and_restart", "test_manager_diagnostics_after_install_retain_backup_without_mutation", "test_manager_diagnostics_after_stop_retain_backup_without_restart", @@ -181,46 +170,17 @@ func TestAutoUpdateServiceStateFailsClosed(t *testing.T) { "test_rollback_keeps_backup_until_restored_running_executable_matches", "test_early_collector_failure_survives_later_success_and_drains_installer", "test_unterminated_final_installer_line_is_collected_and_its_failure_retained", - } - for _, name := range required { - passed := false - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") { - passed = true - } - } - if !passed { - t.Fatalf("required updater collector control %s did not pass:\n%s", name, out) - } - } + }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) } // Keep the full real-sink matrix mandatory, but give it its own existing-size // bound rather than exhausting the service-state suite's manager deadlines. func TestAutoUpdateLogSinkFailures(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_auto_update_service_state.py"), "-v", "UpdateLogSinks") - cmd.Env = append(os.Environ(), "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("unattended real-log sink controls: %v\n%s", err, out) - } - for _, name := range []string{ + runPythonFixtureControls(t, repoFile("scripts", "tests", "test_auto_update_service_state.py"), "UpdateLogSinks", []string{ "test_logger_double_keeps_complete_argument_frames_and_exit_sequence", "test_real_log_retains_each_sink_failure_in_conditional_calls", "test_real_logger_failure_drains_installer_and_preserves_recovery", - } { - passed := false - for _, line := range strings.Split(string(out), "\n") { - if strings.HasPrefix(line, name+" (") && strings.HasSuffix(line, " ... ok") { - passed = true - } - } - if !passed { - t.Fatalf("required real-log sink control %s did not pass:\n%s", name, out) - } - } + }, 45*time.Second, "PULSE_AUTO_UPDATER_UNDER_TEST="+repoFile("scripts", "pulse-auto-update.sh")) } // A real owned process retains its old executable after its disk path changes. diff --git a/scripts/installtests/python_fixture_controls_test.go b/scripts/installtests/python_fixture_controls_test.go new file mode 100644 index 0000000000..e24a56321b --- /dev/null +++ b/scripts/installtests/python_fixture_controls_test.go @@ -0,0 +1,123 @@ +package installtests + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// A suite's cumulative runtime is not a service or installer deadline. Give +// each independent control the existing bound, while the native phase keeps +// its overall Go timeout. Discovery runs every method, including future ones, +// and the required list prevents deleting a safety control unnoticed. +func runPythonFixtureControls(t *testing.T, script, class string, required []string, bound time.Duration, env ...string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), bound) + controls, err := discoverPythonFixtureControls(ctx, script, class, required, env...) + cancel() + if err != nil { + t.Fatal(err) + } + t.Logf("Discovered %d %s controls; each retains its %s bound", len(controls), class, bound) + for _, name := range controls { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + output, err := executePythonFixtureControl(ctx, script, class, name, env...) + if err != nil { + t.Fatal(err) + } + t.Logf("%s", output) + }) + } +} + +func discoverPythonFixtureControls(ctx context.Context, script, class string, required []string, env ...string) ([]string, error) { + const discover = `import json, runpy, sys, unittest +namespace = runpy.run_path(sys.argv[1]) +suite = unittest.defaultTestLoader.loadTestsFromTestCase(namespace[sys.argv[2]]) +print(json.dumps([test._testMethodName for test in suite])) +` + cmd := exec.CommandContext(ctx, "python3", "-I", "-S", "-c", discover, script, class) + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("discover %s controls: %w (context %v)\n%s", class, err, ctx.Err(), output) + } + var controls []string + if err := json.Unmarshal(output, &controls); err != nil { + return nil, fmt.Errorf("decode %s controls: %w\n%s", class, err, output) + } + if len(controls) == 0 { + return nil, fmt.Errorf("no controls discovered in %s", class) + } + for _, name := range required { + found := false + for _, control := range controls { + found = found || control == name + } + if !found { + return nil, fmt.Errorf("required %s control %s is absent from discovery", class, name) + } + } + return controls, nil +} + +func executePythonFixtureControl(ctx context.Context, script, class, name string, env ...string) ([]byte, error) { + selector := class + "." + name + cmd := exec.CommandContext(ctx, "python3", "-I", "-S", script, "-v", selector) + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + if err != nil { + return output, fmt.Errorf("%s: %w (context %v)\n%s", selector, err, ctx.Err(), output) + } + if !strings.Contains(string(output), name+" (__main__."+selector+") ... ok") || + !strings.Contains(string(output), "\nRan 1 test in ") || + !strings.Contains(string(output), "\nOK\n") { + return output, fmt.Errorf("%s did not execute exactly one passing control:\n%s", selector, output) + } + return output, nil +} + +func TestPythonFixtureControlsRetainDiscoveryAndRejectFalsePasses(t *testing.T) { + script := filepath.Join(t.TempDir(), "controls.py") + const source = `import unittest +class Controls(unittest.TestCase): + def test_added(self): pass + def test_fail(self): self.fail("retained adverse control") + def test_pass(self): pass + @unittest.skip("must not be credited") + def test_skip(self): pass +if __name__ == "__main__": unittest.main(verbosity=2) +` + if err := os.WriteFile(script, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + controls, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_pass"}) + if err != nil { + t.Fatal(err) + } + if want := []string{"test_added", "test_fail", "test_pass", "test_skip"}; !reflect.DeepEqual(controls, want) { + t.Fatalf("new controls must be discovered too: got %v, want %v", controls, want) + } + if _, err := discoverPythonFixtureControls(ctx, script, "Controls", []string{"test_removed"}); err == nil { + t.Fatal("a deleted required control must fail discovery") + } + for _, name := range []string{"test_pass", "test_skip", "test_fail", "test_missing"} { + t.Run(name, func(t *testing.T) { + _, err := executePythonFixtureControl(ctx, script, "Controls", name) + if (err == nil) != (name == "test_pass") { + t.Fatalf("execution verdict for %s: %v", name, err) + } + }) + } +}