Adopt the release train: promote the soaked candidate, not the branch tip

Stable promotions built whatever the dispatch branch was at that second.
The resolver checked that HEAD descends from the promoted release
candidate but never that its content matches, so v6.4.0 shipped 64
changed files, including product code, that v6.4.0-rc.12 had not
soaked. Every v6 version was mapped to main, which now moves every few
minutes under the autonomous maintainer, so each fix to a candidate
brought everything landed since and stable was never an exact soaked
commit. Five of six stable minor releases shipped under version-bound
owner exceptions that waived the soak.

From v6.5.0 the release train applies (RELEASE_PROMOTION_POLICY.md,
"Release Train"): a two-week train sized to measured velocity, a
release/v6.N branch per train declared in the control plane so the
workflow refuses a dispatch from anywhere else, a stable promotion that
may differ from its candidate only in release metadata unless
hotfix_exception names active customer harm, and a seven day soak for
minor releases. The 6.4.x line stays on main so the v6.4.3-rc.1
candidate already prepared there is unaffected. The gap is registered
as coverage gap release-train-exact-candidate-promotion.

scripts/release_control/resolve_release_promotion.py

@@ -24,6 +24,28 @@ SEMVER_PUBLISHED_PRERELEASE_RE = re.compile(
 )
 SEMVER_RC_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)-rc\.(\d+)$")
 MIN_PRERELEASE_OBSERVATION_HOURS = 24
+# Release train (RELEASE_PROMOTION_POLICY.md, "Release Train"): a patch keeps
+# the 72 hour candidate soak; a minor release soaks its candidate for a week.
+MIN_STABLE_SOAK_HOURS = 72
+MIN_MINOR_STABLE_SOAK_HOURS = 168
+RELEASE_TRAIN_MIN_VERSION = (6, 5, 0)
+# Paths a stable promotion may change relative to its promoted candidate.
+# Everything else is content the candidate never soaked, so the resolver
+# refuses it unless hotfix_exception names active customer harm.
+RELEASE_METADATA_PATH_RE = re.compile(
+    r"^(?:"
+    r"VERSION"
+    r"|deploy/helm/pulse/(?:Chart\.yaml|README\.md)"
+    r"|docker-compose\.yml"
+    r"|docs/RELEASE_NOTES\.md"
+    r"|docs/UPGRADE_v6\.md"
+    r"|frontend-modern/public/docs/(?:RELEASE_NOTES|UPGRADE_v6)\.md"
+    r"|docs/releases/.+"
+    r"|docs/release-control/v6/internal/records/.+"
+    r"|docs/release-control/v6/internal/status\.json"
+    r"|docs/release-control/v6/internal/subsystems/deployment-installability\.md"
+    r")$"
+)
 WINDOWS_AUTHENTICODE_AVAILABLE = False
 WINDOWS_AUTHENTICODE_STANDING_UNSIGNED_MIN_VERSION = (6, 3, 2)
 WINDOWS_AUTHENTICODE_UNAVAILABLE_REASON = (
@@ -492,11 +514,37 @@ def resolve_metadata(
             promoted_tag_ts = tag_created_unix_fn(promoted_from_tag)
             soak_hours_value = int((now_unix_fn() - promoted_tag_ts) / 3600)
             soak_hours = str(soak_hours_value)
+            # The release train governs v6.5.0 and later. Earlier lines shipped
+            # under the previous regime and their recorded exceptions stand.
+            train_governed = bool(stable_version and stable_version >= RELEASE_TRAIN_MIN_VERSION)
+            candidate_content_drift: list[str] = []
+            if train_governed:
+                candidate_content_drift = [
+                    path
+                    for path in changed_paths_fn(promoted_from_tag)
+                    if not RELEASE_METADATA_PATH_RE.match(path)
+                ]
 
             if hotfix_exception:
                 if not hotfix_reason:
                     raise ValueError("hotfix_reason is required when hotfix_exception is true.")
-            elif soak_hours_value < 72:
+            elif candidate_content_drift:
+                shown = ", ".join(candidate_content_drift[:10])
+                if len(candidate_content_drift) > 10:
+                    shown += f", and {len(candidate_content_drift) - 10} more"
+                raise ValueError(
+                    f"Stable promotion {tag} would ship content that {promoted_from_tag} never soaked "
+                    f"({len(candidate_content_drift)} paths beyond release metadata: {shown}). "
+                    "Cut another release candidate from the release branch, or use hotfix_exception "
+                    "with a concrete active-customer-harm reason."
+                )
+            elif train_governed and not stable_patch and soak_hours_value < MIN_MINOR_STABLE_SOAK_HOURS:
+                raise ValueError(
+                    f"Minor stable promotion {tag} has only {soak_hours_value} hours of prerelease soak since "
+                    f"{promoted_from_tag}; the release train requires {MIN_MINOR_STABLE_SOAK_HOURS} hours "
+                    "(seven days) for a minor release unless hotfix_exception is true."
+                )
+            elif soak_hours_value < MIN_STABLE_SOAK_HOURS:
                 raise ValueError(
                     f"Stable promotion {tag} has only {soak_hours_value} hours of prerelease soak since {promoted_from_tag}; minimum is 72 hours unless hotfix_exception is true."
                 )

scripts/release_control/resolve_release_promotion_test.py

@@ -790,5 +790,100 @@ class ResolveReleasePromotionTest(unittest.TestCase):
             )
 
 
+class ReleaseTrainPromotionTest(unittest.TestCase):
+    """The release train: a stable ships its soaked candidate, and minors soak a week."""
+
+    def promote(self, version: str, **overrides):
+        promoted = f"{version}-rc.1"
+        arguments = dict(
+            version=version,
+            promoted_from_tag_input=promoted,
+            rollback_version_input="6.4.1",
+            ga_date_input="",
+            v5_eos_date_input="",
+            hotfix_exception=False,
+            hotfix_reason_input="",
+            release_notes_input="",
+            tag_exists_fn=lambda tag: tag in {f"v{promoted}", "v6.4.1"},
+            tag_commit_fn=lambda tag: "abc123",
+            head_descends_from_fn=lambda commit: commit == "abc123",
+            tag_created_unix_fn=lambda tag: 100,
+            now_unix_fn=lambda: 100 + (168 * 3600),
+            changed_paths_fn=lambda base_tag: [
+                "VERSION",
+                "deploy/helm/pulse/Chart.yaml",
+                "docs/RELEASE_NOTES.md",
+                "docs/releases/RELEASE_NOTES_v6.5.0.md",
+                "docs/release-control/v6/internal/status.json",
+            ],
+        )
+        arguments.update(overrides)
+        return resolver.resolve_metadata(**arguments)
+
+    def test_release_metadata_paths_are_the_only_allowed_drift(self) -> None:
+        for path in (
+            "VERSION",
+            "deploy/helm/pulse/Chart.yaml",
+            "deploy/helm/pulse/README.md",
+            "docker-compose.yml",
+            "docs/RELEASE_NOTES.md",
+            "docs/UPGRADE_v6.md",
+            "frontend-modern/public/docs/UPGRADE_v6.md",
+            "docs/releases/V6_CHANGELOG_v6.5.0.md",
+            "docs/release-control/v6/internal/records/v6.5.0-ga.md",
+            "docs/release-control/v6/internal/status.json",
+            "docs/release-control/v6/internal/subsystems/deployment-installability.md",
+        ):
+            with self.subTest(path=path):
+                self.assertIsNotNone(resolver.RELEASE_METADATA_PATH_RE.match(path))
+        for path in (
+            "internal/api/router.go",
+            "frontend-modern/src/App.tsx",
+            ".github/workflows/create-release.yml",
+            "docs/TRUENAS.md",
+            "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md",
+            "scripts/install.sh",
+        ):
+            with self.subTest(path=path):
+                self.assertIsNone(resolver.RELEASE_METADATA_PATH_RE.match(path))
+
+    def test_minor_promotion_ships_exactly_the_soaked_candidate(self) -> None:
+        metadata = self.promote("6.5.0")
+        self.assertEqual(metadata["promoted_from_tag"], "v6.5.0-rc.1")
+        self.assertEqual(metadata["soak_hours"], "168")
+
+    def test_content_the_candidate_never_soaked_is_refused(self) -> None:
+        drift = [
+            "VERSION",
+            "internal/api/router.go",
+            "frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx",
+        ]
+        with self.assertRaisesRegex(ValueError, "never soaked.*2 paths beyond release metadata"):
+            self.promote("6.5.0", changed_paths_fn=lambda base_tag: drift)
+        with self.assertRaisesRegex(ValueError, "never soaked"):
+            self.promote("6.5.1", changed_paths_fn=lambda base_tag: drift)
+
+    def test_hotfix_exception_still_requires_a_reason_for_drift(self) -> None:
+        drift = ["VERSION", "internal/api/router.go"]
+        with self.assertRaisesRegex(ValueError, "hotfix_reason is required"):
+            self.promote("6.5.1", hotfix_exception=True, changed_paths_fn=lambda base_tag: drift)
+        metadata = self.promote(
+            "6.5.1",
+            hotfix_exception=True,
+            hotfix_reason_input="Active customer harm: agents cannot re-enrol after upgrade.",
+            changed_paths_fn=lambda base_tag: drift,
+            now_unix_fn=lambda: 100 + (2 * 3600),
+        )
+        self.assertEqual(metadata["hotfix_exception"], "true")
+
+    def test_minor_releases_soak_seven_days_and_patches_seventy_two_hours(self) -> None:
+        with self.assertRaisesRegex(ValueError, "release train requires 168 hours"):
+            self.promote("6.5.0", now_unix_fn=lambda: 100 + (100 * 3600))
+        metadata = self.promote("6.5.1", now_unix_fn=lambda: 100 + (73 * 3600))
+        self.assertEqual(metadata["soak_hours"], "73")
+        with self.assertRaisesRegex(ValueError, "minimum is 72 hours"):
+            self.promote("6.5.1", now_unix_fn=lambda: 100 + (71 * 3600))
+
+
 if __name__ == "__main__":
     unittest.main()