mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-11-07 02:29:54 +00:00
pulse-triage[bot]
a370a6f8de
Spool held test output after 64 KiB per root without truncation, private-name paths or altered coverage verdicts. Preserve failed and unfinished logs while passing and skipped roots stay quiet. Exercise four interleaved roots with 96 MiB of output under a fixed 64 MiB recorder budget, plus Unicode/order and existing real Go race/coverage controls. Change-source: pulse-maintainer Contract-Neutral: Ordinary CI diagnostic storage only; log bytes/order, timing and index schemas, source selection and verdicts are unchanged. Executable controls are in scripts/tests/test_api_failure_index.py.
537 lines
31 KiB
Python
537 lines
31 KiB
Python
#!/usr/bin/env python3
|
|
"""Shard artifacts expose bounded source identities, never assertion payloads."""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RECORDER = ROOT / '.github/scripts/record-internal-api-test-seconds.py'
|
|
|
|
|
|
def event(action, test=None, output=None, elapsed=None):
|
|
value = {'Action': action, 'Package': 'example/internal/api'}
|
|
if test is not None:
|
|
value['Test'] = test
|
|
if output is not None:
|
|
value['Output'] = output
|
|
if elapsed is not None:
|
|
value['Elapsed'] = elapsed
|
|
return json.dumps(value) + '\n'
|
|
|
|
|
|
class APIFailureIndexTest(unittest.TestCase):
|
|
def record(self, stream, selected=None, package='example/internal/api'):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
seconds = Path(directory) / 'api-2.txt'
|
|
command = [sys.executable, str(RECORDER), str(seconds)]
|
|
if selected is not None:
|
|
selection = Path(directory) / 'api-2.selected'
|
|
selection.write_text('\n'.join(selected) + '\n')
|
|
command.extend([str(selection), package])
|
|
result = subprocess.run(command,
|
|
input=stream, text=True, capture_output=True,
|
|
timeout=10)
|
|
raw = Path(str(seconds) + '.failures.json').read_text()
|
|
return result, seconds.read_text(), json.loads(raw), raw
|
|
|
|
def test_spilled_output_keeps_interleaved_unicode_and_unfinished_order(self):
|
|
names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestZPending', 'TestAPending']
|
|
output = {name: ''.join(f'{i}: λ雪 {name}/private-case\n' for i in range(4096))
|
|
for name in names}
|
|
stream = ''.join(event('run', name) for name in names)
|
|
for i in range(0, max(map(len, output.values())), 1024):
|
|
for name in names:
|
|
stream += event('output', name + '/private-case', output[name][i:i + 1024])
|
|
stream += (event('fail', 'TestBroken', elapsed=.1) +
|
|
event('pass', 'TestQuiet', elapsed=.2) +
|
|
event('skip', 'TestSkipped', elapsed=.3) + event('fail', elapsed=.4))
|
|
result, seconds, index, raw = self.record(stream, names)
|
|
self.assertEqual(result.returncode, 1, result.stderr)
|
|
expected = output['TestBroken']
|
|
for name in ['TestAPending', 'TestZPending']:
|
|
expected += f'--- {name} did not finish; its output follows\n' + output[name]
|
|
self.assertEqual(result.stdout, expected)
|
|
self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.10\n'
|
|
'TestQuiet 0.20\nTestSkipped 0.30\n')
|
|
self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])
|
|
self.assertEqual(index['unfinished_top_level_tests']['names'],
|
|
['TestAPending', 'TestZPending'])
|
|
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
|
|
self.assertNotIn('private-case', raw)
|
|
|
|
def noisy_record(self, *, interrupted):
|
|
# Stream 96 MiB from disk in small events: the recorder, not the test
|
|
# driver, must retain all four active roots. No application/Go test or
|
|
# excluded crash scenario is involved. A fixed address-space budget
|
|
# catches output-sized retention without a timing/RSS threshold.
|
|
import resource
|
|
names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestPending']
|
|
chunks = {name: (name + '/private-case λ\r\n').ljust(8192, '.') for name in names}
|
|
repeats = 3072
|
|
expected = hashlib.sha256()
|
|
if interrupted:
|
|
for _ in range(repeats):
|
|
expected.update(chunks['TestBroken'].encode())
|
|
expected.update(b'--- TestPending did not finish; its output follows\n')
|
|
for _ in range(repeats):
|
|
expected.update(chunks['TestPending'].encode())
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
source, seconds, log = (root / name for name in ['events.jsonl', 'seconds.txt', 'log'])
|
|
spool = root / 'spool'
|
|
spool.mkdir()
|
|
selection = root / 'selected'
|
|
selection.write_text('\n'.join(names) + '\n')
|
|
with source.open('w', encoding='utf-8', newline='') as stream:
|
|
stream.write(''.join(event('run', name) for name in names))
|
|
rows = ''.join(event('output', name + '/private-case', chunks[name]) for name in names)
|
|
for _ in range(repeats):
|
|
stream.write(rows)
|
|
stream.write(event('fail' if interrupted else 'pass', names[0], elapsed=.1))
|
|
stream.write(event('pass', names[1], elapsed=.2))
|
|
stream.write(event('skip', names[2], elapsed=.3))
|
|
if not interrupted:
|
|
stream.write(event('pass', names[3], elapsed=.4))
|
|
stream.write(event('fail' if interrupted else 'pass', elapsed=.5))
|
|
|
|
def memory_budget():
|
|
resource.setrlimit(resource.RLIMIT_AS, (64 * 1024**2, 64 * 1024**2))
|
|
|
|
with source.open('rb') as stream, log.open('wb') as output:
|
|
result = subprocess.run(
|
|
[sys.executable, str(RECORDER), str(seconds), str(selection), 'example/internal/api'],
|
|
stdin=stream, stdout=output, stderr=subprocess.PIPE, timeout=60,
|
|
env={**os.environ, 'TMPDIR': str(spool)}, preexec_fn=memory_budget)
|
|
self.assertEqual(result.returncode, int(interrupted), result.stderr.decode())
|
|
self.assertNotIn(b'Traceback', result.stderr)
|
|
actual = hashlib.sha256()
|
|
with log.open('rb') as output:
|
|
for block in iter(lambda: output.read(64 * 1024), b''):
|
|
actual.update(block)
|
|
self.assertEqual(actual.hexdigest(), expected.hexdigest())
|
|
raw = Path(str(seconds) + '.failures.json').read_text()
|
|
index = json.loads(raw)
|
|
self.assertEqual(index['recorder_exit_code'], int(interrupted))
|
|
self.assertEqual(index['completed_top_level_test_count'], 3 if interrupted else 4)
|
|
self.assertEqual(index['unfinished_top_level_tests']['names'],
|
|
['TestPending'] if interrupted else [])
|
|
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
|
|
self.assertNotIn('private-case', raw)
|
|
self.assertEqual(list(spool.iterdir()), [])
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
|
|
def test_noisy_passing_and_skipped_roots_fit_fixed_memory(self):
|
|
self.noisy_record(interrupted=False)
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
|
|
def test_noisy_failed_and_unfinished_roots_fit_fixed_memory_without_log_loss(self):
|
|
self.noisy_record(interrupted=True)
|
|
|
|
def test_pass_preserves_timing_and_quiet_output(self):
|
|
result, seconds, index, _ = self.record(
|
|
event('run', 'TestGood') + event('output', 'TestGood', 'private output\n') +
|
|
event('pass', 'TestGood', elapsed=1.25) + event('pass', elapsed=2))
|
|
self.assertEqual(result.returncode, 0)
|
|
self.assertEqual(result.stdout, '')
|
|
self.assertEqual(seconds, '# package-seconds 2.00\nTestGood 1.25\n')
|
|
self.assertEqual(index['failed_top_level_tests'],
|
|
{'names': [], 'observed_count': 0, 'omitted_count': 0})
|
|
self.assertEqual(index['package_terminal_action'], 'pass')
|
|
self.assertEqual(index['package_terminal_count'], 1)
|
|
|
|
def test_failure_names_root_without_subtest_or_assertion_data(self):
|
|
secret = 'customer@example.invalid'
|
|
result, seconds, index, raw = self.record(
|
|
event('run', 'TestBroken') +
|
|
event('output', 'TestBroken/' + secret, secret + '\n') +
|
|
event('fail', 'TestBroken/' + secret, elapsed=0.2) +
|
|
event('fail', 'TestBroken', elapsed=0.3) + event('fail', elapsed=0.4))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn(secret, result.stdout) # Existing diagnostic log stays intact.
|
|
self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.30\n')
|
|
self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])
|
|
self.assertEqual(index['unfinished_top_level_tests']['names'], [])
|
|
self.assertEqual(index['package_terminal_action'], 'fail')
|
|
self.assertNotIn(secret, raw)
|
|
self.assertNotIn('Package', raw)
|
|
|
|
def test_unfinished_root_is_not_a_completed_failure_or_pass(self):
|
|
result, seconds, index, raw = self.record(
|
|
event('run', 'TestInterrupted') +
|
|
event('output', 'TestInterrupted/private-label', 'private log\n'))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('TestInterrupted did not finish', result.stdout)
|
|
self.assertEqual(seconds, '')
|
|
self.assertEqual(index['failed_top_level_tests']['names'], [])
|
|
self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestInterrupted'])
|
|
self.assertIsNone(index['package_terminal_action'])
|
|
self.assertEqual(index['package_terminal_count'], 0)
|
|
self.assertNotIn('private', raw)
|
|
|
|
def test_package_only_failure_does_not_invent_a_test_identity(self):
|
|
result, _, index, raw = self.record(
|
|
event('output', output='private compiler diagnostic\n') + event('fail'))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('compiler diagnostic', result.stdout)
|
|
self.assertEqual(index['failed_top_level_tests']['names'], [])
|
|
self.assertEqual(index['package_terminal_action'], 'fail')
|
|
self.assertNotIn('diagnostic', raw)
|
|
|
|
def test_non_json_and_empty_streams_remain_unknown(self):
|
|
for stream, count in [('', 0), ('private non-json line\n[]\n', 2)]:
|
|
with self.subTest(stream=bool(stream)):
|
|
result, _, index, raw = self.record(stream)
|
|
# Unknown completion cannot become a passing workflow verdict,
|
|
# even if a producer happens to return zero.
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['non_json_line_count'], count)
|
|
self.assertIsNone(index['package_terminal_action'])
|
|
self.assertNotIn('private', raw)
|
|
|
|
def test_unsafe_and_long_source_names_are_counted_but_not_exported(self):
|
|
for name in ['TestUnsafe\n::error::private', 'TestUnsafe\x1b[31m',
|
|
'TestÜnicode', 'not-a-source-test', 'Test' + 'A' * 257]:
|
|
with self.subTest(name=name[:12]):
|
|
result, _, index, raw = self.record(event('fail', name))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['failed_top_level_tests'],
|
|
{'names': [], 'observed_count': 1, 'omitted_count': 1})
|
|
self.assertNotIn(name, raw)
|
|
|
|
def test_lists_are_deduplicated_sorted_and_bounded_separately(self):
|
|
stream = ''.join(event('fail', f'TestFailed{i:04d}') for i in range(300, -1, -1))
|
|
stream += event('fail', 'TestFailed0000')
|
|
stream += ''.join(event('run', f'TestPending{i:04d}') for i in range(300, -1, -1))
|
|
result, _, index, raw = self.record(stream)
|
|
self.assertEqual(result.returncode, 1)
|
|
for field in ['failed_top_level_tests', 'unfinished_top_level_tests']:
|
|
value = index[field]
|
|
self.assertEqual(len(value['names']), 256)
|
|
self.assertEqual(value['observed_count'], 301)
|
|
self.assertEqual(value['omitted_count'], 45)
|
|
self.assertEqual(value['names'], sorted(set(value['names'])))
|
|
self.assertLess(len(raw.encode()), 150_000)
|
|
|
|
def test_package_actions_preserve_contrary_observations_without_green_claim(self):
|
|
result, _, index, _ = self.record(event('fail') + event('pass'))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['package_terminal_action'], 'pass')
|
|
self.assertEqual(index['package_terminal_count'], 2)
|
|
self.assertNotIn('success', index)
|
|
self.assertEqual(index['recorder_exit_code'], 1)
|
|
|
|
def test_skip_and_supported_source_name_forms(self):
|
|
stream = ''.join(event('fail', name) for name in
|
|
['Test', 'TestUnder_score123', 'Example', 'ExampleThing', 'FuzzThing'])
|
|
result, _, index, _ = self.record(stream + event('skip'))
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['failed_top_level_tests']['observed_count'], 5)
|
|
self.assertEqual(index['failed_top_level_tests']['omitted_count'], 0)
|
|
self.assertEqual(index['package_terminal_action'], 'skip')
|
|
|
|
def test_sidecar_is_ignored_by_the_existing_timing_refresh(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
seconds = root / 'api-2.txt'
|
|
(root / 'api-2.selected').write_text('TestTimed\nTestSelectionIsNotATiming\n')
|
|
result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],
|
|
input=event('run', 'TestTimed') +
|
|
event('pass', 'TestTimed', elapsed=1.25) +
|
|
event('pass', elapsed=2),
|
|
text=True, capture_output=True, timeout=10)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
weights = root / 'weights.out'
|
|
result = subprocess.run(
|
|
[sys.executable, str(ROOT / '.github/scripts/refresh-internal-api-test-seconds.py'),
|
|
'--dir', str(root), '--output', str(weights)],
|
|
text=True, capture_output=True, timeout=10)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn('TestTimed 1.25', weights.read_text())
|
|
self.assertNotIn('failures.json', result.stdout)
|
|
|
|
def test_pipefail_preserves_an_upstream_failure_with_no_json(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
seconds = Path(directory) / 'api-2.txt'
|
|
result = subprocess.run(
|
|
['bash', '-c', 'set -o pipefail; (printf "compile diagnostic\\n"; exit 17) | '
|
|
'python3 "$1" "$2"', 'recorder-proof', str(RECORDER), str(seconds)],
|
|
text=True, capture_output=True, timeout=10)
|
|
# Both producer and recorder fail; pipefail returns the last
|
|
# failing stage rather than manufacturing success.
|
|
self.assertEqual(result.returncode, 1)
|
|
index = json.loads(Path(str(seconds) + '.failures.json').read_text())
|
|
self.assertIsNone(index['package_terminal_action'])
|
|
self.assertEqual(index['recorder_exit_code'], 1)
|
|
self.assertEqual(index['non_json_line_count'], 1)
|
|
|
|
def test_existing_upload_collects_sidecar_on_a_failed_shard(self):
|
|
workflow = (ROOT / '.github/workflows/build-and-test.yml').read_text()
|
|
job = workflow.split('\n backend-api:\n', 1)[1].split('\n backend-api-verdict:', 1)[0]
|
|
self.assertIn('| python3 .github/scripts/record-internal-api-test-seconds.py', job)
|
|
self.assertIn('if: always() && needs.changes.outputs.code ==', job)
|
|
self.assertIn('path: ${{ runner.temp }}/internal-api-test-seconds/', job)
|
|
self.assertIn('name: internal-api-test-seconds-${{ matrix.index }}', job)
|
|
self.assertIn('set -euo pipefail', job)
|
|
self.assertNotIn('continue-on-error:', job)
|
|
self.assertIn('printf \'%s\\n\' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"', job)
|
|
self.assertIn('package=$(go list ./internal/api)', job)
|
|
self.assertIn('"$timings/api-${API_SHARD_INDEX}.selected" "$package"', job)
|
|
|
|
def test_selected_pass_and_explicit_skip_complete_coverage(self):
|
|
stream = (event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2) +
|
|
event('run', 'TestNative') + event('skip', 'TestNative', elapsed=0) +
|
|
event('pass', elapsed=0.3))
|
|
result, _, index, _ = self.record(stream, ['TestGood', 'TestNative'])
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(index['selected_test_count'], 2)
|
|
self.assertEqual(index['completed_top_level_test_count'], 2)
|
|
self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
|
|
self.assertEqual(index['unexpected_top_level_tests']['observed_count'], 0)
|
|
self.assertIn('skipped_top_level_tests', index)
|
|
self.assertEqual(index['skipped_top_level_tests'],
|
|
{'names': ['TestNative'], 'observed_count': 1, 'omitted_count': 0})
|
|
self.assertEqual(index['top_level_tests_with_skipped_subtests']['observed_count'], 0)
|
|
self.assertEqual(index['skipped_subtest_event_count'], 0)
|
|
|
|
def test_skipped_subtests_do_not_disappear_under_a_passing_parent(self):
|
|
private = 'customer@example.invalid/secret-case'
|
|
stream = (event('run', 'TestPartial') +
|
|
event('run', 'TestPartial/' + private) +
|
|
event('output', 'TestPartial/' + private, 'private skip reason\n') +
|
|
event('skip', 'TestPartial/' + private) +
|
|
event('run', 'TestPartial/another-private-case') +
|
|
event('skip', 'TestPartial/another-private-case') +
|
|
event('pass', 'TestPartial', elapsed=0.5) + event('pass', elapsed=0.6))
|
|
result, seconds, index, raw = self.record(stream, ['TestPartial'])
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(result.stdout, '')
|
|
self.assertEqual(seconds, '# package-seconds 0.60\nTestPartial 0.50\n')
|
|
self.assertIn('top_level_tests_with_skipped_subtests', index)
|
|
self.assertEqual(index['top_level_tests_with_skipped_subtests'],
|
|
{'names': ['TestPartial'], 'observed_count': 1, 'omitted_count': 0})
|
|
self.assertEqual(index['skipped_top_level_tests']['observed_count'], 0)
|
|
self.assertEqual(index['skipped_subtest_event_count'], 2)
|
|
for hidden in [private, 'another-private-case', 'private skip reason']:
|
|
self.assertNotIn(hidden, raw)
|
|
|
|
def test_skip_observations_survive_failure_and_missing_completion(self):
|
|
partial = (event('run', 'TestPartial') +
|
|
event('skip', 'TestPartial/private-case') +
|
|
event('run', 'TestUnavailable') + event('skip', 'TestUnavailable'))
|
|
for terminal in ['', event('fail', 'TestPartial') + event('fail')]:
|
|
with self.subTest(completed=bool(terminal)):
|
|
result, _, index, raw = self.record(
|
|
partial + terminal, ['TestPartial', 'TestUnavailable'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('skipped_top_level_tests', index)
|
|
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestUnavailable'])
|
|
self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'],
|
|
['TestPartial'])
|
|
self.assertEqual(index['skipped_subtest_event_count'], 1)
|
|
self.assertNotIn('private-case', raw)
|
|
field = 'failed_top_level_tests' if terminal else 'unfinished_top_level_tests'
|
|
self.assertEqual(index[field]['names'], ['TestPartial'])
|
|
|
|
def test_skip_identifiers_are_bounded_and_private_labels_are_not_exported(self):
|
|
names = [f'TestSkip{i:04d}' for i in range(300, -1, -1)]
|
|
names += ['TestÜnicode', 'Test' + 'A' * 257, 'not-a-source-test']
|
|
stream = ''.join(event('run', name) + event('skip', name + '/private-child') +
|
|
event('skip', name) for name in names) + event('pass')
|
|
result, _, index, raw = self.record(stream)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
for field in ['skipped_top_level_tests', 'top_level_tests_with_skipped_subtests']:
|
|
self.assertIn(field, index)
|
|
observed = index[field]
|
|
self.assertEqual(observed['observed_count'], 304)
|
|
self.assertEqual(len(observed['names']), 256)
|
|
self.assertEqual(observed['omitted_count'], 48)
|
|
self.assertEqual(observed['names'], sorted(set(observed['names'])))
|
|
self.assertEqual(index['skipped_subtest_event_count'], 304)
|
|
for hidden in ['private-child', 'Ünicode', 'not-a-source-test', 'A' * 257]:
|
|
self.assertNotIn(hidden, raw)
|
|
self.assertLess(len(raw.encode()), 150_000)
|
|
|
|
def test_successful_producer_cannot_substitute_incomplete_execution(self):
|
|
good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
|
|
for stream in ['', event('pass'), good, 'malformed\n' + good + event('pass'),
|
|
good + event('pass') + event('pass'),
|
|
event('pass', 'TestGood') + event('pass'),
|
|
good + good + event('pass'), good + event('skip')]:
|
|
with self.subTest(stream=stream):
|
|
result, _, index, _ = self.record(stream, ['TestGood'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['recorder_exit_code'], 1)
|
|
self.assertIn('no passing verdict', result.stderr)
|
|
|
|
def test_selected_tests_must_not_be_dropped_or_replaced(self):
|
|
good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
|
|
result, _, index, _ = self.record(good + event('pass'), ['TestGood', 'TestMissing'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['missing_selected_tests']['names'], ['TestMissing'])
|
|
other = event('run', 'TestOther') + event('pass', 'TestOther', elapsed=0.2)
|
|
result, _, index, _ = self.record(other + event('pass'), ['TestGood'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['missing_selected_tests']['names'], ['TestGood'])
|
|
self.assertEqual(index['unexpected_top_level_tests']['names'], ['TestOther'])
|
|
|
|
def test_unfinished_selected_test_retains_both_coverage_and_original_log(self):
|
|
stream = event('run', 'TestHung') + event('output', 'TestHung/private', 'private log\n')
|
|
result, _, index, raw = self.record(stream, ['TestHung'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('private log', result.stdout)
|
|
self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestHung'])
|
|
self.assertEqual(index['missing_selected_tests']['names'], ['TestHung'])
|
|
self.assertNotIn('private', raw)
|
|
|
|
def test_wrong_or_multiple_packages_cannot_satisfy_selection(self):
|
|
good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
|
|
for stream, package in [(good, 'other/private-package'),
|
|
(good.replace('example/internal/api', 'other/private-package', 1),
|
|
'example/internal/api')]:
|
|
result, _, index, raw = self.record(stream, ['TestGood'], package)
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertGreater(index['unexpected_package_event_count'], 0)
|
|
self.assertNotIn('private-package', raw)
|
|
|
|
def test_invalid_event_shapes_are_non_passing_not_recorder_crashes(self):
|
|
for changed in [{'Action': 'invented'}, {'Package': None}, {'Test': []}, {'Output': 1}]:
|
|
value = {'Action': 'output', 'Package': 'example/internal/api'}
|
|
value.update(changed)
|
|
stream = (json.dumps(value) + '\n' + event('run', 'TestGood') +
|
|
event('pass', 'TestGood') + event('pass'))
|
|
result, _, index, _ = self.record(stream, ['TestGood'])
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertGreater(index['invalid_event_count'], 0)
|
|
self.assertNotIn('Traceback', result.stderr)
|
|
|
|
def test_go_build_diagnostics_remain_visible_without_test_package_identity(self):
|
|
good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
|
|
for action, code in [('build-output', 0), ('build-fail', 1)]:
|
|
build = json.dumps({'Action': action, 'ImportPath': 'private/dependency',
|
|
'Output': 'retained build diagnostic\n'}) + '\n'
|
|
result, _, index, raw = self.record(build + good, ['TestGood'])
|
|
self.assertEqual(result.returncode, code)
|
|
self.assertIn('retained build diagnostic', result.stdout)
|
|
self.assertEqual(index['package_identity_count'], 1)
|
|
self.assertNotIn('private/dependency', raw)
|
|
self.assertNotIn('diagnostic', raw)
|
|
|
|
def test_coverage_names_stay_bounded_without_rejecting_unicode_source(self):
|
|
missing = [f'TestMissing{i:04d}' for i in range(301)]
|
|
missing.extend(['TestÜnicode', 'Test' + 'A' * 257])
|
|
stream = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
|
|
result, _, index, raw = self.record(stream, ['TestGood'] + missing)
|
|
self.assertEqual(result.returncode, 1)
|
|
observation = index['missing_selected_tests']
|
|
self.assertEqual(observation['observed_count'], 303)
|
|
self.assertEqual(len(observation['names']), 256)
|
|
self.assertEqual(observation['omitted_count'], 47)
|
|
self.assertNotIn('Ünicode', raw)
|
|
result, _, _, _ = self.record(
|
|
event('run', 'TestÜnicode') + event('pass', 'TestÜnicode') + event('pass'),
|
|
['TestÜnicode'])
|
|
self.assertEqual(result.returncode, 0)
|
|
|
|
def test_invalid_selection_is_rejected_before_observation(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
seconds = Path(directory) / 'api.txt'
|
|
selected = Path(directory) / 'api.selected'
|
|
for names in ['', 'TestA\nTestA\n', 'TestA/subtest\n', 'TestBad-name\n']:
|
|
selected.write_text(names)
|
|
result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
|
|
str(selected), 'example/internal/api'],
|
|
input='', text=True, capture_output=True, timeout=10)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertFalse(seconds.exists())
|
|
selected.unlink()
|
|
result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
|
|
str(selected), 'example/internal/api'],
|
|
input='', text=True, capture_output=True, timeout=10)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn('unavailable', result.stderr)
|
|
|
|
@unittest.skipUnless(shutil.which('go'), 'real Go JSON requires the declared compiler')
|
|
def test_real_race_json_pass_skip_fuzz_example_failure_and_missing_coverage(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
package = 'example.invalid/shardfixture'
|
|
(root / 'go.mod').write_text(f'module {package}\n\ngo 1.24\n')
|
|
(root / 'fixture_test.go').write_text('''package shardfixture
|
|
import ("fmt"; "testing")
|
|
type Fixture struct{}
|
|
func TestGood(t *testing.T) { t.Log("passing fixture output") }
|
|
func TestSkip(t *testing.T) { t.Skip("explicit native fixture exclusion") }
|
|
func TestPartial(t *testing.T) {
|
|
t.Run("working", func(t *testing.T) {})
|
|
t.Run("private-child", func(t *testing.T) { t.Skip("private native fixture reason") })
|
|
}
|
|
func TestFailure(t *testing.T) { t.Fatal("retained fixture failure") }
|
|
func FuzzSeed(f *testing.F) { f.Add(1); f.Fuzz(func(t *testing.T, n int) {}) }
|
|
func ExampleFixture() { fmt.Println("fixture")
|
|
// Output: fixture
|
|
}
|
|
''')
|
|
env = {**os.environ, 'GOWORK': 'off', 'GOPROXY': 'off', 'GOTOOLCHAIN': 'local'}
|
|
listing = subprocess.run(['go', 'test', '-race', '-list', '.', '.'],
|
|
cwd=root, env=env, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(listing.returncode, 0, listing.stderr)
|
|
names = [name for name in listing.stdout.splitlines()
|
|
if name.startswith(('Test', 'Fuzz', 'Example'))]
|
|
selected = [name for name in names if name != 'TestFailure']
|
|
self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'TestPartial', 'FuzzSeed', 'ExampleFixture'})
|
|
command = ['go', 'test', '-race', '-count=1', '-timeout', '30s', '-json']
|
|
# Exercise the same shorter-side -skip path used by large CI shards.
|
|
passing = subprocess.run(command + ['-skip', '^TestFailure$', '.'],
|
|
cwd=root, env=env, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(passing.returncode, 0, passing.stderr)
|
|
result, _, index, _ = self.record(passing.stdout, selected, package)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(index['completed_top_level_test_count'], 5)
|
|
self.assertIn('skipped_top_level_tests', index)
|
|
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkip'])
|
|
self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'], ['TestPartial'])
|
|
self.assertEqual(index['skipped_subtest_event_count'], 1)
|
|
self.assertNotIn('private-child', json.dumps(index))
|
|
self.assertNotIn('private native fixture reason', json.dumps(index))
|
|
self.assertNotIn('passing fixture output', result.stdout)
|
|
|
|
# Real Go exits zero while selecting fewer tests than the source
|
|
# list expects. Neither pipefail nor a package pass detects that.
|
|
omitted = subprocess.run(command + ['-run', '^TestGood$', '.'],
|
|
cwd=root, env=env, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(omitted.returncode, 0, omitted.stderr)
|
|
result, _, index, _ = self.record(omitted.stdout, selected, package)
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertEqual(index['missing_selected_tests']['observed_count'], 4)
|
|
|
|
# Completed tests without the final package event are not a pass.
|
|
events = [json.loads(line) for line in passing.stdout.splitlines()]
|
|
truncated = ''.join(json.dumps(value) + '\n' for value in events
|
|
if not (value.get('Action') == 'pass' and 'Test' not in value))
|
|
result, _, index, _ = self.record(truncated, selected, package)
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIsNone(index['package_terminal_action'])
|
|
|
|
failing = subprocess.run(command + ['-run', '^TestFailure$', '.'],
|
|
cwd=root, env=env, text=True, capture_output=True, timeout=120)
|
|
self.assertEqual(failing.returncode, 1, failing.stderr)
|
|
result, _, index, _ = self.record(failing.stdout, ['TestFailure'], package)
|
|
self.assertEqual(result.returncode, 1)
|
|
self.assertIn('retained fixture failure', result.stdout)
|
|
self.assertEqual(index['failed_top_level_tests']['names'], ['TestFailure'])
|
|
self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
|
|
print('Real Go race JSON: complete pass/skip/fuzz/example accepted; '
|
|
'producer-zero omission, truncated completion and actual failure rejected.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|