mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 18:13:57 +00:00
ci(validation): wire persistence and lock source plans
This commit is contained in:
cxymds
@@ -40,11 +40,11 @@ on:
|
||||
required: false
|
||||
default: ''
|
||||
i05_diagnostic_branch:
|
||||
description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle
|
||||
description: I05 source-plan runs both H1/H2 controllers with exact native evidence; subsets remain diagnostic
|
||||
type: choice
|
||||
options: [timeout, queued-successor]
|
||||
options: [source-plan, timeout, queued-successor]
|
||||
required: false
|
||||
default: timeout
|
||||
default: source-plan
|
||||
auto_testing_sha:
|
||||
description: Exact auto-testing commit for a validation case
|
||||
type: string
|
||||
@@ -325,8 +325,8 @@ jobs:
|
||||
raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup")
|
||||
if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"):
|
||||
raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution")
|
||||
diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout"
|
||||
if diagnostic_branch not in {"timeout", "queued-successor"}:
|
||||
diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "source-plan"
|
||||
if diagnostic_branch not in {"source-plan", "timeout", "queued-successor"}:
|
||||
raise ValueError("I05 diagnostic branch is not registered")
|
||||
if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort":
|
||||
raise ValueError("queued-successor is available only for the I05 diagnostic case")
|
||||
@@ -453,7 +453,7 @@ jobs:
|
||||
printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Download pinned native storage evidence
|
||||
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }}
|
||||
if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout') || (inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac') }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
@@ -508,18 +508,34 @@ jobs:
|
||||
> "${VALIDATION_ROOT}/native-race-runtime-preflight.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Verify I05 queued-successor native prerequisite
|
||||
if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }}
|
||||
- name: Verify I05 source-plan or queued-successor native prerequisites
|
||||
if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout' }}
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }}
|
||||
I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
if [ "${I05_DIAGNOSTIC_BRANCH}" = 'queued-successor' ]; then
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
|
||||
> "${VALIDATION_ROOT}/native-i05-prerequisite.json"
|
||||
else
|
||||
python3 -B auto-testing/scripts/validation_heal/lock_source.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" \
|
||||
> "${VALIDATION_ROOT}/native-i05-source-prerequisites.json"
|
||||
fi
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Bind F07 exact-candidate directory fsync evidence
|
||||
if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json"
|
||||
python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \
|
||||
--directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \
|
||||
--testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \
|
||||
> "${VALIDATION_ROOT}/native-i05-prerequisite.json"
|
||||
printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Download pinned manager persistence and incarnation evidence
|
||||
|
||||
@@ -196,34 +196,64 @@ class RaceNativeToolsWorkflowTests(unittest.TestCase):
|
||||
self.assertIn("inputs.validation_case == 'heal.race-multipart-complete-abort'", selection)
|
||||
|
||||
def test_i05_native_prerequisite_propagates_failure_and_exports_only_after_verification(self):
|
||||
shell = self.chain_shell_step('Verify I05 queued-successor native prerequisite')
|
||||
for exit_code in (0, 42):
|
||||
with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as name:
|
||||
root = Path(name)
|
||||
env = self.fixture(root, new_helper=True)
|
||||
script = root / 'auto-testing/scripts/validation_heal/race_native_evidence.py'
|
||||
script.write_text('import os,sys\n'
|
||||
'assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n'
|
||||
'assert sys.argv[-3] == os.environ["AUTO_TESTING_SHA"]\n'
|
||||
'sys.exit(int(os.environ["VERIFY_EXIT"]))\n')
|
||||
validation = root / 'validation'; validation.mkdir()
|
||||
(validation / 'tool-run.json').write_text('{"id":123}')
|
||||
native = root / 'rustfs-validation-native-77-1'; native.mkdir()
|
||||
output = root / 'env'
|
||||
env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
|
||||
VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code))
|
||||
result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, exit_code, result.stderr)
|
||||
self.assertEqual(output.exists(), exit_code == 0)
|
||||
self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
|
||||
shell = self.chain_shell_step('Verify I05 source-plan or queued-successor native prerequisites')
|
||||
for branch in ('source-plan', 'queued-successor'):
|
||||
for exit_code in (0, 42):
|
||||
with self.subTest(branch=branch, exit_code=exit_code), tempfile.TemporaryDirectory() as name:
|
||||
root = Path(name)
|
||||
env = self.fixture(root, new_helper=True)
|
||||
script_name = 'lock_source.py' if branch == 'source-plan' else 'race_native_evidence.py'
|
||||
script = root / 'auto-testing/scripts/validation_heal' / script_name
|
||||
script.write_text('import os,sys\n'
|
||||
'assert sys.argv[1] == "--directory"\n'
|
||||
'assert "--candidate" in sys.argv\n'
|
||||
'assert sys.argv[sys.argv.index("--testing-commit") + 1] == os.environ["AUTO_TESTING_SHA"]\n'
|
||||
'if os.environ["I05_DIAGNOSTIC_BRANCH"] == "queued-successor":\n'
|
||||
' assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n'
|
||||
'else:\n'
|
||||
' assert "--case-id" not in sys.argv\n'
|
||||
'sys.exit(int(os.environ["VERIFY_EXIT"]))\n')
|
||||
validation = root / 'validation'; validation.mkdir()
|
||||
(validation / 'tool-run.json').write_text('{"id":123}')
|
||||
native = root / 'rustfs-validation-native-77-1'; native.mkdir()
|
||||
output = root / 'env'
|
||||
env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
|
||||
VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code),
|
||||
I05_DIAGNOSTIC_BRANCH=branch)
|
||||
result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, exit_code, result.stderr)
|
||||
self.assertEqual(output.exists(), exit_code == 0)
|
||||
self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
|
||||
workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
|
||||
self.assertLess(workflow.index('Verify I05 queued-successor native prerequisite'),
|
||||
self.assertLess(workflow.index('Verify I05 source-plan or queued-successor native prerequisites'),
|
||||
workflow.index('Run selected canonical validation case'))
|
||||
download = workflow.split(' - name: Download pinned native storage evidence\n', 1)[1].split(' uses:', 1)[0]
|
||||
self.assertIn("inputs.i05_diagnostic_branch == 'queued-successor'", download)
|
||||
self.assertIn("inputs.i05_diagnostic_branch != 'timeout'", download)
|
||||
execution = workflow.split(' - name: Run selected canonical validation case\n', 1)[1].split(' run:', 1)[0]
|
||||
self.assertIn('I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}', execution)
|
||||
|
||||
def test_f07_directory_fsync_artifact_is_candidate_scoped_and_bound(self):
|
||||
shell = self.chain_shell_step('Bind F07 exact-candidate directory fsync evidence')
|
||||
with tempfile.TemporaryDirectory() as name:
|
||||
root = Path(name)
|
||||
validation = root / 'validation'; validation.mkdir()
|
||||
(validation / 'tool-run.json').write_text('{"id":123}')
|
||||
native = root / 'rustfs-validation-native-77-1'; native.mkdir()
|
||||
output = root / 'env'
|
||||
env = dict(os.environ, RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1',
|
||||
VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output))
|
||||
result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes())
|
||||
self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n')
|
||||
workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text()
|
||||
binding = workflow.split(' - name: Bind F07 exact-candidate directory fsync evidence\n', 1)[1]
|
||||
guard = binding.split(' run:', 1)[0]
|
||||
self.assertIn("inputs.validation_case == 'heal.checkpoint-and-mrf-corruption'", guard)
|
||||
self.assertIn("steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac'", guard)
|
||||
self.assertLess(workflow.index('Bind F07 exact-candidate directory fsync evidence'),
|
||||
workflow.index('Run selected canonical validation case'))
|
||||
|
||||
def shell_step(self, name):
|
||||
from check_test_wiring import yaml_block
|
||||
source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()
|
||||
|
||||
Reference in New Issue
Block a user