mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 14:00:29 +00:00
pulse-triage[bot]
b116f2defc
The held latency failure lacks contemporaneous resource context, and isolated passing samples do not explain it. Retain allowlisted backend boundary counters including cgroup ancestors in the existing durable log, preserving backend failures and unchanged qualification thresholds. Focused tests cover failure exits, unavailable telemetry, ancestor collection and environment filtering. Change-source: pulse-maintainer
590 lines
26 KiB
Python
Executable File
590 lines
26 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[3]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from shard_go_tests import _test_regex, allocate_cpu_plan, build_plan, write_plan
|
|
|
|
|
|
class ReleasePreflightTest(unittest.TestCase):
|
|
def test_backend_cpu_plan_reserves_non_api_capacity_without_oversubscription(
|
|
self,
|
|
) -> None:
|
|
widths, non_api = allocate_cpu_plan([3624, 15, 16], 8)
|
|
self.assertEqual(widths, [4, 1, 1])
|
|
self.assertEqual(non_api, 2)
|
|
self.assertEqual(sum(widths) + non_api, 8)
|
|
|
|
# Stable releases use a four-vCPU hosted runner. Keep the measured
|
|
# three-way API partition there as 1/1/1 plus one non-API worker;
|
|
# falling back to two equal-count shards exceeded the 45-minute
|
|
# watchdog in stable v6.4.0 run 33220050114.
|
|
widths, non_api = allocate_cpu_plan([3624, 15, 16], 4)
|
|
self.assertEqual(widths, [1, 1, 1])
|
|
self.assertEqual(non_api, 1)
|
|
self.assertEqual(sum(widths) + non_api, 4)
|
|
|
|
widths, non_api = allocate_cpu_plan([10, 10], 4)
|
|
self.assertEqual(widths, [1, 1])
|
|
self.assertEqual(non_api, 2)
|
|
self.assertEqual(sum(widths) + non_api, 4)
|
|
|
|
widths, non_api = allocate_cpu_plan([10], 1)
|
|
self.assertEqual(widths, [1])
|
|
self.assertEqual(non_api, 0)
|
|
|
|
for counts, vcpus in [([], 8), ([1], 0), ([1, 1], 1), ([0], 1)]:
|
|
with self.subTest(counts=counts, vcpus=vcpus):
|
|
with self.assertRaises(ValueError):
|
|
allocate_cpu_plan(counts, vcpus)
|
|
|
|
def run_script(
|
|
self, *args: str, env: dict[str, str] | None = None
|
|
) -> subprocess.CompletedProcess[str]:
|
|
command_env = os.environ.copy()
|
|
if env:
|
|
command_env.update(env)
|
|
return subprocess.run(
|
|
[str(ROOT / "scripts/run-release-preflight.sh"), *args],
|
|
cwd=ROOT,
|
|
env=command_env,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
def test_plan_resolves_exact_sha_and_wsl_transport(self) -> None:
|
|
sha = subprocess.check_output(
|
|
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True
|
|
).strip()
|
|
result = self.run_script(
|
|
"--profile",
|
|
"rehearsal",
|
|
"--host",
|
|
"test-worker",
|
|
"--wsl-distro",
|
|
"Ubuntu",
|
|
"--plan",
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn(f"SHA: {sha}", result.stdout)
|
|
self.assertIn("Runtime: WSL Ubuntu", result.stdout)
|
|
|
|
def test_missing_optional_worker_is_a_non_gate(self) -> None:
|
|
configured_host = subprocess.run(
|
|
["git", "config", "--get", "pulse.releasePreflightHost"],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if configured_host.returncode == 0:
|
|
self.skipTest("repository has a local release-preflight host configured")
|
|
result = self.run_script(
|
|
"--profile",
|
|
"release",
|
|
"--if-configured",
|
|
env={
|
|
"PULSE_RELEASE_PREFLIGHT_HOST": "",
|
|
"PULSE_RELEASE_PREFLIGHT_WSL_DISTRO": "",
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
},
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("canonical hosted checks", result.stdout)
|
|
|
|
def test_worker_rejects_non_exact_sha_before_touching_worker_state(self) -> None:
|
|
result = subprocess.run(
|
|
[str(ROOT / "scripts/release-preflight-worker.sh"), "HEAD", "rehearsal"],
|
|
cwd=ROOT,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn("40-character Git commit id", result.stderr)
|
|
|
|
def test_dispatch_helpers_select_the_matching_profiles(self) -> None:
|
|
dry_run = (ROOT / "scripts/trigger-release-dry-run.sh").read_text()
|
|
release = (ROOT / "scripts/trigger-release.sh").read_text()
|
|
workflow = (ROOT / ".github/workflows/release-dry-run.yml").read_text()
|
|
self.assertIn("--profile rehearsal", dry_run)
|
|
self.assertIn("--profile release", release)
|
|
self.assertIn("--if-configured", dry_run)
|
|
self.assertIn("--if-configured", release)
|
|
self.assertIn('PULSE_E2E_DIAGNOSTIC: "1"', workflow)
|
|
|
|
def assert_release_dry_run_diagnostic_contract(
|
|
self, workflow: str, diagnostic: str
|
|
) -> None:
|
|
installed_browser = re.search(
|
|
r"npx playwright install --with-deps ([a-z-]+)", workflow
|
|
)
|
|
diagnostic_step = re.search(
|
|
r" - name: Run integration diagnostics\n(?P<body>.*?)(?=\n - name:)",
|
|
workflow,
|
|
flags=re.DOTALL,
|
|
)
|
|
self.assertIsNotNone(diagnostic_step)
|
|
step_body = diagnostic_step.group("body")
|
|
diagnostic_command = re.search(
|
|
r"npx playwright test tests/00-diagnostic\.spec\.ts (?P<args>[^\n]+)",
|
|
step_body,
|
|
)
|
|
self.assertIsNotNone(installed_browser)
|
|
self.assertIsNotNone(diagnostic_command)
|
|
self.assertIn('PULSE_E2E_DIAGNOSTIC: "1"', step_body)
|
|
self.assertLess(
|
|
step_body.index("set -o pipefail"),
|
|
step_body.index("npx playwright test tests/00-diagnostic.spec.ts"),
|
|
)
|
|
self.assertIn("2>&1 | tee diagnostic-evidence/playwright-diagnostic.log", step_body)
|
|
selected_browser = re.search(
|
|
r"--project(?:=|\s+)([a-z-]+)", diagnostic_command.group("args")
|
|
)
|
|
self.assertIsNotNone(
|
|
selected_browser,
|
|
"release diagnostic must select the one browser installed by the job",
|
|
)
|
|
self.assertEqual(selected_browser.group(1), installed_browser.group(1))
|
|
self.assertIn("--retries=0", diagnostic_command.group("args"))
|
|
|
|
self.assertRegex(
|
|
diagnostic,
|
|
r"test\.describe\.configure\(\{\s*retries:\s*0\s*\}\);",
|
|
)
|
|
self.assertIn("test.setTimeout(120_000)", diagnostic)
|
|
|
|
for required_diagnostic in (
|
|
"API readiness failed:",
|
|
"dependencies.monitor",
|
|
"dependencies.scheduler",
|
|
"dependencies.websocket",
|
|
"/api/security/status",
|
|
"hasAuthentication",
|
|
"Rendered UI readiness failed:",
|
|
"Pulse app shell did not render within 20s",
|
|
"effectiveRenderedOpacity",
|
|
"stayed transparent",
|
|
"Pulse Setup Wizard",
|
|
"Paste your bootstrap token",
|
|
"diagnostic-evidence",
|
|
"release-dry-run-rendered-readiness-chromium.png",
|
|
"testInfo.attach(",
|
|
"DIAGNOSTIC EVIDENCE ERROR:",
|
|
"if (!readinessFailed)",
|
|
):
|
|
self.assertIn(required_diagnostic, diagnostic)
|
|
|
|
self.assertIn("Collect integration diagnostic runtime evidence", workflow)
|
|
self.assertIn("Upload integration diagnostic evidence", workflow)
|
|
self.assertIn("if-no-files-found: error", workflow)
|
|
self.assertIn("tests/integration/test-results/", workflow)
|
|
self.assertNotIn("expect(true).toBe(true)", diagnostic)
|
|
self.assertNotIn("await page.waitForTimeout(3000)", diagnostic)
|
|
|
|
def test_release_dry_run_diagnostic_contract(self) -> None:
|
|
workflow = (ROOT / ".github/workflows/release-dry-run.yml").read_text()
|
|
diagnostic = (
|
|
ROOT / "tests/integration/tests/00-diagnostic.spec.ts"
|
|
).read_text()
|
|
|
|
self.assert_release_dry_run_diagnostic_contract(workflow, diagnostic)
|
|
|
|
legacy_workflow = workflow.replace(
|
|
" --project=chromium --retries=0", "", 1
|
|
)
|
|
with self.assertRaises(AssertionError):
|
|
self.assert_release_dry_run_diagnostic_contract(
|
|
legacy_workflow, diagnostic
|
|
)
|
|
|
|
retrying_diagnostic = diagnostic.replace(
|
|
"test.describe.configure({ retries: 0 });", "", 1
|
|
)
|
|
with self.assertRaises(AssertionError):
|
|
self.assert_release_dry_run_diagnostic_contract(
|
|
workflow, retrying_diagnostic
|
|
)
|
|
|
|
unconditional_pass = f"{diagnostic}\nexpect(true).toBe(true);\n"
|
|
with self.assertRaises(AssertionError):
|
|
self.assert_release_dry_run_diagnostic_contract(
|
|
workflow, unconditional_pass
|
|
)
|
|
|
|
evidence_failure_ignored = diagnostic.replace(
|
|
"if (!readinessFailed) {", "if (false) {", 1
|
|
)
|
|
with self.assertRaises(AssertionError):
|
|
self.assert_release_dry_run_diagnostic_contract(
|
|
workflow, evidence_failure_ignored
|
|
)
|
|
|
|
def test_worker_has_no_publication_or_signing_authority(self) -> None:
|
|
worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
|
|
runner = (ROOT / "scripts/run-release-preflight.sh").read_text()
|
|
self.assertIn("unset GH_TOKEN GITHUB_TOKEN", worker)
|
|
self.assertIn("export GITHUB_ACTIONS=true", worker)
|
|
self.assertIn("export CI=true", worker)
|
|
self.assertIn('export GOTMPDIR="$GO_TMP_DIR"', worker)
|
|
self.assertIn('rm -rf "$GO_TMP_DIR"', worker)
|
|
self.assertIn('sub(/^toolchain /, "")', worker)
|
|
self.assertNotIn("docker push", worker)
|
|
self.assertNotIn("gh release", worker)
|
|
self.assertIn("mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble", worker)
|
|
self.assertIn(
|
|
'git show "${SOURCE_SHA}:scripts/release-preflight-worker.sh"', runner
|
|
)
|
|
self.assertIn("is not reachable from a fetched origin branch", runner)
|
|
|
|
def test_worker_matches_ci_toolchain_and_publishes_smoke_on_a_free_port(self) -> None:
|
|
worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
|
|
# Non-login ssh shells never source profile hooks, so an installed mise
|
|
# toolchain must be activated explicitly or a stale system Node wins.
|
|
self.assertIn('"$HOME/.local/bin/mise" activate bash --shims', worker)
|
|
# GitHub runners leave GOTMPDIR unset; unix-socket fixtures sit at the
|
|
# sun_path limit under /tmp, so the worker must not nest test temp dirs.
|
|
self.assertIn('GO_TMP_DIR="${PULSE_RELEASE_PREFLIGHT_GO_TMP_DIR:-}"', worker)
|
|
# Smoke stacks publish on a verified-free host port pair and every probe
|
|
# and the browser use that port; a worker may host Pulse on 7655.
|
|
self.assertIn("smoke_port_pair_is_free", worker)
|
|
self.assertIn("PULSE_RELEASE_PREFLIGHT_E2E_PORT", worker)
|
|
self.assertIn('export PULSE_E2E_BASE_URL="http://localhost:${PULSE_E2E_PORT}"', worker)
|
|
self.assertIn('--env "PLAYWRIGHT_BASE_URL=${PULSE_E2E_BASE_URL}"', worker)
|
|
self.assertNotIn("localhost:7655", worker)
|
|
|
|
def test_worker_serializes_resource_intensive_test_suites(self) -> None:
|
|
worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
|
|
scheduling_block = re.search(
|
|
r"# Static frontend checks.*?run_backend\n",
|
|
worker,
|
|
flags=re.DOTALL,
|
|
)
|
|
self.assertIsNotNone(scheduling_block)
|
|
block = scheduling_block.group(0)
|
|
|
|
self.assertIn("run_frontend_static_quality &", block)
|
|
self.assertIn("run_integration_prep &", block)
|
|
self.assertNotIn("run_frontend_tests &", block)
|
|
self.assertNotIn("run_backend &", block)
|
|
self.assertLess(block.index("done\n"), block.index("run_frontend_tests\n"))
|
|
self.assertLess(block.index("run_frontend_tests\n"), block.index("run_backend\n"))
|
|
|
|
def test_backend_preserves_exit_and_collects_after_failure(self):
|
|
worker = (ROOT / 'scripts/release-preflight-worker.sh').read_text()
|
|
function = re.search(r'^run_backend\(\) \(\n.*?^\)', worker, re.M | re.S).group(0)
|
|
for profile in ('release', 'rehearsal'):
|
|
for code in (0, 17):
|
|
for telemetry_code in (0, 9):
|
|
with self.subTest(profile=profile, code=code, telemetry_code=telemetry_code):
|
|
# Stub only operations: execute the real backend shell function
|
|
# under errexit, without any product tests or data mutation.
|
|
script = f'''set -euo pipefail
|
|
PROFILE={profile}
|
|
TEST_DATA_DIR=unused
|
|
ACTUAL_GO=go-test-fixture
|
|
rm() {{ :; }}
|
|
mkdir() {{ :; }}
|
|
python3() {{ echo "snapshot $2"; return {telemetry_code}; }}
|
|
phase() {{ echo "phase $1"; return {code}; }}
|
|
{function}
|
|
run_backend
|
|
echo CONTINUED
|
|
'''
|
|
result = subprocess.run(['bash', '-c', script], text=True, capture_output=True)
|
|
self.assertEqual(result.returncode, code, result.stderr)
|
|
self.assertIn('snapshot before', result.stdout)
|
|
self.assertIn('snapshot after', result.stdout)
|
|
self.assertIn(f'RELEASE_BACKEND_EXIT {code}', result.stdout)
|
|
self.assertEqual('CONTINUED' in result.stdout, code == 0)
|
|
self.assertIn('backend-serial' if profile == 'rehearsal' else 'backend-race-sharded', result.stdout)
|
|
|
|
|
|
def test_browser_mount_identity_and_locked_cli_follow_the_docker_daemon(self) -> None:
|
|
worker = (ROOT / "scripts/release-preflight-worker.sh").read_text()
|
|
functions = []
|
|
for name in ("playwright_container_user", "run_playwright"):
|
|
match = re.search(rf"^{name}\(\) \{{\n.*?^\}}", worker, re.MULTILINE | re.DOTALL)
|
|
self.assertIsNotNone(match, name)
|
|
functions.append(match.group(0))
|
|
for options, info_rc, valid, expected_user in (
|
|
(["name=seccomp,profile=builtin", "name=rootless"], 0, True, "0:0"),
|
|
(["name=seccomp,profile=builtin"], 0, True, f"{os.getuid()}:{os.getgid()}"),
|
|
([], 42, False, None),
|
|
({"rootless": True}, 0, False, None),
|
|
(None, 0, False, None),
|
|
):
|
|
with self.subTest(options=options, info_rc=info_rc), tempfile.TemporaryDirectory() as raw:
|
|
temp = pathlib.Path(raw)
|
|
docker = temp / "docker"
|
|
docker.write_text(
|
|
f"#!{sys.executable}\n"
|
|
"import json,os,pathlib,sys\n"
|
|
"if sys.argv[1] == 'info':\n"
|
|
" print(os.environ['TEST_SECURITY_OPTIONS'])\n"
|
|
" raise SystemExit(int(os.environ['TEST_INFO_RC']))\n"
|
|
"pathlib.Path(os.environ['TEST_CONTAINER_STARTED']).touch()\n"
|
|
"print(json.dumps(sys.argv[1:]))\n"
|
|
)
|
|
docker.chmod(0o755)
|
|
started = temp / "started"
|
|
script = "set -euo pipefail\n" + "\n".join(functions) + "\n"
|
|
script += 'PLAYWRIGHT_CONTAINER_USER="$(playwright_container_user)"\n'
|
|
script += "run_playwright --version\n"
|
|
script += "run_playwright test tests/00-diagnostic.spec.ts --project=chromium\n"
|
|
result = subprocess.run(
|
|
["bash", "-c", script], capture_output=True, text=True, check=False,
|
|
env={**os.environ, "PATH": f"{temp}:{os.environ['PATH']}",
|
|
"TEST_SECURITY_OPTIONS": json.dumps(options), "TEST_INFO_RC": str(info_rc),
|
|
"TEST_CONTAINER_STARTED": str(started),
|
|
"REPOSITORY_DIR": str(temp / "private workspace"),
|
|
"PLAYWRIGHT_IMAGE": "locked-playwright-image",
|
|
"PULSE_E2E_BASE_URL": "http://127.0.0.1:27655"},
|
|
)
|
|
self.assertEqual(valid, result.returncode == 0, result.stderr)
|
|
self.assertEqual(valid, started.exists())
|
|
if not valid:
|
|
continue
|
|
calls = [json.loads(line) for line in result.stdout.splitlines()]
|
|
self.assertEqual(2, len(calls))
|
|
for args in calls:
|
|
self.assertEqual(expected_user, args[args.index("--user") + 1])
|
|
self.assertEqual(f"{temp}/private workspace/tests/integration:/work",
|
|
args[args.index("--volume") + 1])
|
|
self.assertIn("node", args)
|
|
self.assertIn("/work/node_modules/@playwright/test/cli.js", args)
|
|
self.assertNotIn("npx", args)
|
|
self.assertEqual("--version", calls[0][-1])
|
|
self.assertEqual(["test", "tests/00-diagnostic.spec.ts", "--project=chromium"], calls[1][-3:])
|
|
prep = worker[worker.index("run_integration_prep() {"):worker.index("# Static frontend checks")]
|
|
self.assertLess(prep.index("phase playwright-image"), prep.index("phase playwright-runtime"))
|
|
self.assertIn("phase playwright-runtime run_playwright --version", prep)
|
|
|
|
def test_api_shard_plan_is_deterministic_complete_and_disjoint(self) -> None:
|
|
test_names = [
|
|
f"TestReleaseCase{index:04d}"
|
|
for index in [8, 1, 14, 3, 16, 0, 7, 11, 2, 15, 5, 13, 4, 10, 6, 12, 9]
|
|
]
|
|
plan = build_plan(test_names, shard_count=2, batch_size=4)
|
|
reordered_plan = build_plan(list(reversed(test_names)), shard_count=2, batch_size=4)
|
|
self.assertNotEqual(plan, reordered_plan)
|
|
|
|
assigned = [
|
|
name
|
|
for shard in plan["shards"]
|
|
for batch in shard["batches"]
|
|
for name in batch
|
|
]
|
|
self.assertEqual(assigned, test_names)
|
|
self.assertEqual(len(assigned), len(set(assigned)))
|
|
self.assertEqual(
|
|
[name for batch in plan["shards"][0]["batches"] for name in batch],
|
|
test_names[:9],
|
|
)
|
|
self.assertEqual(
|
|
[name for batch in plan["shards"][1]["batches"] for name in batch],
|
|
test_names[9:],
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
manifest_path = write_plan(plan, pathlib.Path(directory))
|
|
manifest = manifest_path.read_text()
|
|
self.assertIn('"test_count": 17', manifest)
|
|
self.assertTrue((pathlib.Path(directory) / "shard-01-batch-01.regex").is_file())
|
|
self.assertTrue((pathlib.Path(directory) / "shard-02-batch-01.regex").is_file())
|
|
|
|
def test_api_shard_plan_rejects_invalid_or_duplicate_names(self) -> None:
|
|
with self.assertRaisesRegex(ValueError, "invalid top-level"):
|
|
build_plan(["TestGood", "BenchmarkNotATest"], 2, 10)
|
|
with self.assertRaisesRegex(ValueError, "duplicate top-level"):
|
|
build_plan(["TestDuplicate", "TestDuplicate"], 2, 10)
|
|
with self.assertRaisesRegex(ValueError, "cannot exceed"):
|
|
build_plan(["TestOnly"], 2, 10)
|
|
with self.assertRaisesRegex(ValueError, "safe per-argument ceiling"):
|
|
build_plan(["TestOnly"], 1, 10, max_regex_bytes=120_001)
|
|
with self.assertRaisesRegex(ValueError, "one value per shard"):
|
|
build_plan(["TestOne", "TestTwo"], 2, 10, shard_weights=[1])
|
|
with self.assertRaisesRegex(ValueError, "positive integers"):
|
|
build_plan(["TestOne", "TestTwo"], 2, 10, shard_weights=[1, 0])
|
|
with self.assertRaisesRegex(ValueError, "one fewer value"):
|
|
build_plan(
|
|
["TestOne", "TestTwo"],
|
|
2,
|
|
10,
|
|
shard_boundaries=["TestOne", "TestTwo"],
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "unknown shard boundary"):
|
|
build_plan(
|
|
["TestOne", "TestTwo"],
|
|
2,
|
|
10,
|
|
shard_boundaries=["TestMissing"],
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "mutually exclusive"):
|
|
build_plan(
|
|
["TestOne", "TestTwo"],
|
|
2,
|
|
10,
|
|
shard_weights=[1, 1],
|
|
shard_boundaries=["TestOne"],
|
|
)
|
|
|
|
def test_api_shard_plan_supports_measured_contiguous_weights(self) -> None:
|
|
test_names = [f"TestReleaseCase{index:04d}" for index in range(20)]
|
|
plan = build_plan(
|
|
test_names,
|
|
shard_count=3,
|
|
batch_size=100,
|
|
shard_weights=[6, 1, 1],
|
|
)
|
|
|
|
shards = [
|
|
[name for batch in shard["batches"] for name in batch]
|
|
for shard in plan["shards"]
|
|
]
|
|
self.assertEqual([len(shard) for shard in shards], [15, 3, 2])
|
|
self.assertEqual([name for shard in shards for name in shard], test_names)
|
|
self.assertEqual(plan["shard_weights"], [6, 1, 1])
|
|
|
|
def test_api_shard_plan_supports_named_contiguous_boundaries(self) -> None:
|
|
test_names = [f"TestReleaseCase{index:04d}" for index in range(8)]
|
|
plan = build_plan(
|
|
test_names,
|
|
shard_count=3,
|
|
batch_size=100,
|
|
shard_boundaries=[test_names[3], test_names[5]],
|
|
)
|
|
|
|
shards = [
|
|
[name for batch in shard["batches"] for name in batch]
|
|
for shard in plan["shards"]
|
|
]
|
|
self.assertEqual([len(shard) for shard in shards], [4, 2, 2])
|
|
self.assertEqual([name for shard in shards for name in shard], test_names)
|
|
self.assertEqual(
|
|
plan["shard_boundaries"], [test_names[3], test_names[5]]
|
|
)
|
|
|
|
def test_api_shard_plan_bounds_one_shard_regex_argument_bytes(self) -> None:
|
|
test_names = [
|
|
f"Test{index:04d}{index * 7919:08d}{index * 104729:010d}{'X' * 64}"
|
|
for index in range(3736)
|
|
]
|
|
max_regex_bytes = 64 * 1024
|
|
plan = build_plan(
|
|
test_names,
|
|
shard_count=1,
|
|
batch_size=10000,
|
|
max_regex_bytes=max_regex_bytes,
|
|
)
|
|
batches = plan["shards"][0]["batches"]
|
|
self.assertGreater(len(batches), 1)
|
|
self.assertEqual(
|
|
[name for batch in batches for name in batch],
|
|
test_names,
|
|
)
|
|
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
write_plan(plan, pathlib.Path(directory))
|
|
regex_files = sorted(pathlib.Path(directory).glob("*.regex"))
|
|
self.assertEqual(len(regex_files), len(batches))
|
|
for regex_file in regex_files:
|
|
self.assertLessEqual(
|
|
len(regex_file.read_text().strip().encode()),
|
|
max_regex_bytes,
|
|
)
|
|
|
|
with self.assertRaisesRegex(ValueError, "exceeds max regex bytes"):
|
|
build_plan(
|
|
["TestNameThatCannotFit"],
|
|
shard_count=1,
|
|
batch_size=1,
|
|
max_regex_bytes=8,
|
|
)
|
|
|
|
def test_api_shard_regex_compresses_shared_names_without_changing_membership(
|
|
self,
|
|
) -> None:
|
|
test_names = [
|
|
"TestHandleCharts_Success",
|
|
"TestHandleCharts_MethodNotAllowed",
|
|
"TestHandleStorageCharts_Success",
|
|
"TestHandleStorageCharts_MethodNotAllowed",
|
|
"TestRouterPublicPathsInventory",
|
|
]
|
|
regex = _test_regex(test_names)
|
|
naive = "^(?:" + "|".join(test_names) + ")$"
|
|
self.assertLess(len(regex.encode()), len(naive.encode()))
|
|
|
|
compiled = re.compile(regex)
|
|
for test_name in test_names:
|
|
self.assertIsNotNone(compiled.fullmatch(test_name))
|
|
for excluded in [
|
|
"TestHandleCharts",
|
|
"TestHandleCharts_Failure",
|
|
"TestRouterPublicPathsInventoryExtra",
|
|
]:
|
|
self.assertIsNone(compiled.fullmatch(excluded))
|
|
|
|
def test_release_workflow_uses_independent_pve_bundle_and_backend_lanes(self) -> None:
|
|
workflow = (ROOT / ".github/workflows/create-release.yml").read_text()
|
|
backend = (ROOT / "scripts/run-release-backend-tests.sh").read_text()
|
|
self.assertIn("frontend_bundle:", workflow)
|
|
bundle_job = workflow[
|
|
workflow.index(" frontend_bundle:") : workflow.index(" frontend_checks:")
|
|
]
|
|
self.assertIn('"pulse-pve-build"', bundle_job)
|
|
self.assertIn("./scripts/run-release-backend-tests.sh", workflow)
|
|
backend_job = workflow[
|
|
workflow.index(" backend_tests:") : workflow.index(" integration_tests:")
|
|
]
|
|
self.assertIn('"pulse-pve-tests"', backend_job)
|
|
self.assertNotIn(" - frontend_checks\n if:", backend_job)
|
|
self.assertIn("go test -c -race", backend)
|
|
self.assertIn("python3 scripts/shard_go_tests.py", backend)
|
|
self.assertIn('--max-regex-bytes "$MAX_REGEX_BYTES"', backend)
|
|
self.assertIn(
|
|
'MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"',
|
|
backend,
|
|
)
|
|
self.assertIn("shard_admission_required_kib", backend)
|
|
self.assertIn("Degrading to $cpu_shards API shard(s)", backend)
|
|
self.assertIn('if [ "$VCPUS" -ge 4 ]; then', backend)
|
|
self.assertIn("cpu_shards=3", backend)
|
|
self.assertIn(
|
|
'API_SHARD_TIMEOUT="${PULSE_BACKEND_API_SHARD_TIMEOUT:-45m}"',
|
|
backend,
|
|
)
|
|
self.assertIn("RESERVED_OTHER_PACKAGE_PROCS", backend)
|
|
self.assertIn(
|
|
'GOMAXPROCS=1 PULSE_DATA_DIR="$RUN_ROOT/data/other"', backend
|
|
)
|
|
self.assertIn(
|
|
'go test -race -p "$OTHER_PACKAGE_PROCS" -timeout 30m', backend
|
|
)
|
|
self.assertIn('-test.timeout "$API_SHARD_TIMEOUT"', backend)
|
|
self.assertIn(
|
|
"TestWebSocketOriginAllowsTrustedForwardedHostedOriginIPv6Loopback,"
|
|
"TestServerInfoEndpointMethodNotAllowed",
|
|
backend,
|
|
)
|
|
self.assertIn("Completed internal/api shard", backend)
|
|
self.assertIn("export GITHUB_ACTIONS=true", backend)
|
|
self.assertIn("export CI=true", backend)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|