diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index a562ca827..453a87956 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -40,11 +40,11 @@ on: required: false default: '' i05_diagnostic_branch: - description: I05 diagnostic-only branch; queued-successor requires the exact native evidence bundle + description: I05 source-plan runs both H1/H2 controllers with exact native evidence; subsets remain diagnostic type: choice - options: [timeout, queued-successor] + options: [source-plan, timeout, queued-successor] required: false - default: timeout + default: source-plan auto_testing_sha: description: Exact auto-testing commit for a validation case type: string @@ -325,8 +325,8 @@ jobs: raise ValueError("manager regressions require tools-only execution, without a fleet case or cleanup") if cleanup and (not cleanup.isdigit() or int(cleanup) <= 0 or os.environ["SELECTED_CASE"] or os.environ["TOOLS_ONLY"] == "true"): raise ValueError("cleanup requires one exact failed run and is exclusive with case/tool execution") - diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "timeout" - if diagnostic_branch not in {"timeout", "queued-successor"}: + diagnostic_branch = os.environ["I05_DIAGNOSTIC_BRANCH"] or "source-plan" + if diagnostic_branch not in {"source-plan", "timeout", "queued-successor"}: raise ValueError("I05 diagnostic branch is not registered") if diagnostic_branch == "queued-successor" and os.environ["SELECTED_CASE"] != "heal.lock-timeout-and-caller-abort": raise ValueError("queued-successor is available only for the I05 diagnostic case") @@ -453,7 +453,7 @@ jobs: printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}" - name: Download pinned native storage evidence - if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor') }} + if: ${{ inputs.validation_case == 'heal.conflicting-subquorum-authority' || inputs.validation_case == 'heal.bucket-incarnation-recreation' || steps.race_native.outputs.required == 'true' || (inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout') || (inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac') }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: github-token: ${{ github.token }} @@ -508,18 +508,34 @@ jobs: > "${VALIDATION_ROOT}/native-race-runtime-preflight.json" printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" - - name: Verify I05 queued-successor native prerequisite - if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch == 'queued-successor' }} + - name: Verify I05 source-plan or queued-successor native prerequisites + if: ${{ inputs.validation_case == 'heal.lock-timeout-and-caller-abort' && inputs.i05_diagnostic_branch != 'timeout' }} env: AUTO_TESTING_SHA: ${{ inputs.auto_testing_sha }} + I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }} + run: | + set -euo pipefail + NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" + if [ "${I05_DIAGNOSTIC_BRANCH}" = 'queued-successor' ]; then + python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \ + > "${VALIDATION_ROOT}/native-i05-prerequisite.json" + else + python3 -B auto-testing/scripts/validation_heal/lock_source.py \ + --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ + --testing-commit "${AUTO_TESTING_SHA}" \ + > "${VALIDATION_ROOT}/native-i05-source-prerequisites.json" + fi + printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" + + - name: Bind F07 exact-candidate directory fsync evidence + if: ${{ inputs.validation_case == 'heal.checkpoint-and-mrf-corruption' && steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac' }} run: | set -euo pipefail NATIVE_DIRECTORY="${RUNNER_TEMP}/rustfs-validation-native-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" cp "${VALIDATION_ROOT}/tool-run.json" "${NATIVE_DIRECTORY}/tool-run.json" - python3 -B auto-testing/scripts/validation_heal/race_native_evidence.py \ - --directory "${NATIVE_DIRECTORY}" --candidate "${VALIDATION_ROOT}/candidate.json" \ - --testing-commit "${AUTO_TESTING_SHA}" --case-id heal.lock-timeout-and-caller-abort \ - > "${VALIDATION_ROOT}/native-i05-prerequisite.json" printf 'VALIDATION_HEAL_NATIVE_DIRECTORY=%s\n' "${NATIVE_DIRECTORY}" >> "${GITHUB_ENV}" - name: Download pinned manager persistence and incarnation evidence diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index c896683f1..bc145528c 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -196,34 +196,64 @@ class RaceNativeToolsWorkflowTests(unittest.TestCase): self.assertIn("inputs.validation_case == 'heal.race-multipart-complete-abort'", selection) def test_i05_native_prerequisite_propagates_failure_and_exports_only_after_verification(self): - shell = self.chain_shell_step('Verify I05 queued-successor native prerequisite') - for exit_code in (0, 42): - with self.subTest(exit_code=exit_code), tempfile.TemporaryDirectory() as name: - root = Path(name) - env = self.fixture(root, new_helper=True) - script = root / 'auto-testing/scripts/validation_heal/race_native_evidence.py' - script.write_text('import os,sys\n' - 'assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n' - 'assert sys.argv[-3] == os.environ["AUTO_TESTING_SHA"]\n' - 'sys.exit(int(os.environ["VERIFY_EXIT"]))\n') - validation = root / 'validation'; validation.mkdir() - (validation / 'tool-run.json').write_text('{"id":123}') - native = root / 'rustfs-validation-native-77-1'; native.mkdir() - output = root / 'env' - env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', - VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code)) - result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) - self.assertEqual(result.returncode, exit_code, result.stderr) - self.assertEqual(output.exists(), exit_code == 0) - self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) + shell = self.chain_shell_step('Verify I05 source-plan or queued-successor native prerequisites') + for branch in ('source-plan', 'queued-successor'): + for exit_code in (0, 42): + with self.subTest(branch=branch, exit_code=exit_code), tempfile.TemporaryDirectory() as name: + root = Path(name) + env = self.fixture(root, new_helper=True) + script_name = 'lock_source.py' if branch == 'source-plan' else 'race_native_evidence.py' + script = root / 'auto-testing/scripts/validation_heal' / script_name + script.write_text('import os,sys\n' + 'assert sys.argv[1] == "--directory"\n' + 'assert "--candidate" in sys.argv\n' + 'assert sys.argv[sys.argv.index("--testing-commit") + 1] == os.environ["AUTO_TESTING_SHA"]\n' + 'if os.environ["I05_DIAGNOSTIC_BRANCH"] == "queued-successor":\n' + ' assert sys.argv[-2:] == ["--case-id", "heal.lock-timeout-and-caller-abort"]\n' + 'else:\n' + ' assert "--case-id" not in sys.argv\n' + 'sys.exit(int(os.environ["VERIFY_EXIT"]))\n') + validation = root / 'validation'; validation.mkdir() + (validation / 'tool-run.json').write_text('{"id":123}') + native = root / 'rustfs-validation-native-77-1'; native.mkdir() + output = root / 'env' + env.update(RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output), VERIFY_EXIT=str(exit_code), + I05_DIAGNOSTIC_BRANCH=branch) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, exit_code, result.stderr) + self.assertEqual(output.exists(), exit_code == 0) + self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() - self.assertLess(workflow.index('Verify I05 queued-successor native prerequisite'), + self.assertLess(workflow.index('Verify I05 source-plan or queued-successor native prerequisites'), workflow.index('Run selected canonical validation case')) download = workflow.split(' - name: Download pinned native storage evidence\n', 1)[1].split(' uses:', 1)[0] - self.assertIn("inputs.i05_diagnostic_branch == 'queued-successor'", download) + self.assertIn("inputs.i05_diagnostic_branch != 'timeout'", download) execution = workflow.split(' - name: Run selected canonical validation case\n', 1)[1].split(' run:', 1)[0] self.assertIn('I05_DIAGNOSTIC_BRANCH: ${{ inputs.i05_diagnostic_branch }}', execution) + def test_f07_directory_fsync_artifact_is_candidate_scoped_and_bound(self): + shell = self.chain_shell_step('Bind F07 exact-candidate directory fsync evidence') + with tempfile.TemporaryDirectory() as name: + root = Path(name) + validation = root / 'validation'; validation.mkdir() + (validation / 'tool-run.json').write_text('{"id":123}') + native = root / 'rustfs-validation-native-77-1'; native.mkdir() + output = root / 'env' + env = dict(os.environ, RUNNER_TEMP=str(root), GITHUB_RUN_ID='77', GITHUB_RUN_ATTEMPT='1', + VALIDATION_ROOT=str(validation), GITHUB_ENV=str(output)) + result = subprocess.run(['bash', '-c', shell], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((native / 'tool-run.json').read_bytes(), (validation / 'tool-run.json').read_bytes()) + self.assertEqual(output.read_text(), 'VALIDATION_HEAL_NATIVE_DIRECTORY=' + str(native) + '\n') + workflow = (candidate.ROOT / '.github/workflows/rustfs-functional-chain.yml').read_text() + binding = workflow.split(' - name: Bind F07 exact-candidate directory fsync evidence\n', 1)[1] + guard = binding.split(' run:', 1)[0] + self.assertIn("inputs.validation_case == 'heal.checkpoint-and-mrf-corruption'", guard) + self.assertIn("steps.candidate.outputs.source_sha == '1aa9be87d18cf941e5020f19e8622c26ba4b10ac'", guard) + self.assertLess(workflow.index('Bind F07 exact-candidate directory fsync evidence'), + workflow.index('Run selected canonical validation case')) + def shell_step(self, name): from check_test_wiring import yaml_block source = (candidate.ROOT / '.github/workflows/rustfs-validation-tools.yml').read_text()