From a1ae0fa07f2b52c110f95ff98d2a5376e56afbed Mon Sep 17 00:00:00 2001 From: Pulse Test Date: Fri, 28 Aug 2026 20:11:04 +0100 Subject: [PATCH] fix(release): require visual selection evidence --- .github/workflows/create-release.yml | 9 +++---- .../subsystems/deployment-installability.md | 5 ++++ scripts/generate-release-notes.sh | 5 +++- .../installtests/build_release_assets_test.go | 24 +++++++++++++++++++ .../release_control/release_note_visuals.py | 12 +++++++--- .../release_note_visuals_test.py | 14 +++++++++++ scripts/trigger-release.sh | 19 ++++++++------- scripts/trigger-stable-patch.sh | 16 +++++-------- 8 files changed, 77 insertions(+), 27 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index a6b1b2bce..48e4db785 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -14,9 +14,8 @@ on: type: string release_screenshot_plan: description: 'Validated model-selected release-note visual plan (JSON)' - required: false + required: true type: string - default: '{"schema_version":1,"captures":[]}' promoted_from_tag: description: 'Stable only: prerelease tag being promoted (for example 6.0.0-rc.2)' required: false @@ -160,7 +159,8 @@ jobs: PLAN_FILE=$(mktemp) if ! jq -er '.inputs.release_screenshot_plan | select(type == "string" and length > 0)' \ "$GITHUB_EVENT_PATH" > "$PLAN_FILE"; then - printf '%s\n' '{"schema_version":1,"captures":[]}' > "$PLAN_FILE" + echo "::error::release_screenshot_plan must contain an evidence-backed visual decision" + exit 1 fi python3 scripts/release_control/release_note_visuals.py \ validate --plan "$PLAN_FILE" --output "$PLAN_FILE" @@ -797,7 +797,8 @@ jobs: VISUAL_MARKDOWN_FILE=$(mktemp) if ! jq -er '.inputs.release_screenshot_plan | select(type == "string" and length > 0)' \ "$GITHUB_EVENT_PATH" > "$VISUAL_PLAN_FILE"; then - printf '%s\n' '{"schema_version":1,"captures":[]}' > "$VISUAL_PLAN_FILE" + echo "::error::release_screenshot_plan must contain an evidence-backed visual decision" + exit 1 fi python3 scripts/release_control/release_note_visuals.py render \ --plan "$VISUAL_PLAN_FILE" \ diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 14c9a2934..d63068cd5 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -65,6 +65,11 @@ visual selection. Selected before and now images are staged as draft release assets, linked from a `See the difference` section, and must be publicly retrievable before the activation marker commits publication. A current-only image is permitted when a truthful before view is not available. +Canonical publish triggers rerun visual evidence discovery and selection for the +exact notes and comparison range being dispatched. A committed visual sidecar is +review material only and cannot substitute for that run. The model may still +select zero captures when its investigation finds that screenshots add no +meaningful customer value. Customer-facing notes use one outcome list for features and fixes. Each visible change is described once under `What's improved`; a parallel `Fixes` section is diff --git a/scripts/generate-release-notes.sh b/scripts/generate-release-notes.sh index 7fac61a78..727200a17 100755 --- a/scripts/generate-release-notes.sh +++ b/scripts/generate-release-notes.sh @@ -274,6 +274,7 @@ current HEAD with identical generated demo data. Return only JSON in this shape, with at most three captures: { "schema_version": 1, + "decision": "Why the selected views improve the notes, or why no screenshot adds meaningful value", "captures": [ { "id": "lower-case-hyphenated-id", @@ -311,7 +312,9 @@ testid. Actions may be click or wait. Role locators use role and name. Other locators use value. Every state needs a ready locator for content that must be visible in the finished image. Locator names and values are literal accessible text, not regular expressions. Use labels verified against the deterministic -generated demo data. Use no semicolon or em dash characters in public text. +generated demo data. The decision field records the model's evidence-based +reason for selecting these captures or selecting none. Use no semicolon or em +dash characters in public text. Customer release notes: diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index a67909119..a2229333a 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -3206,6 +3206,30 @@ func TestReleaseNotesGeneratorResolvesChannelSpecificComparisonRanges(t *testing } } +func TestReleaseTriggersReevaluateVisualsInsteadOfTrustingSidecars(t *testing.T) { + for _, path := range []string{ + repoFile("scripts", "trigger-release.sh"), + repoFile("scripts", "trigger-stable-patch.sh"), + } { + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + text := string(content) + if strings.Contains(text, "VISUAL_PLAN_SIDECAR") { + t.Fatalf("%s must not treat a committed visual sidecar as dispatch evidence", path) + } + for _, required := range []string{ + "A committed sidecar is review material, not proof", + "generate-release-notes.sh --visual-plan", + } { + if !strings.Contains(text, required) { + t.Fatalf("%s missing visual reevaluation contract %q", path, required) + } + } + } +} + func assertFileContainsAllNormalized(t *testing.T, path string, required ...string) { t.Helper() content, err := os.ReadFile(path) diff --git a/scripts/release_control/release_note_visuals.py b/scripts/release_control/release_note_visuals.py index b6bf544ac..9d0bc99ae 100755 --- a/scripts/release_control/release_note_visuals.py +++ b/scripts/release_control/release_note_visuals.py @@ -131,11 +131,12 @@ def _state(value: Any, field: str) -> dict[str, Any]: def validate_plan(raw: Any) -> dict[str, Any]: if not isinstance(raw, dict): raise PlanError("visual plan must be a JSON object") - unknown = set(raw) - {"schema_version", "captures"} + unknown = set(raw) - {"schema_version", "decision", "captures"} if unknown: raise PlanError(f"visual plan has unsupported fields: {', '.join(sorted(unknown))}") if raw.get("schema_version") != 1: raise PlanError("visual plan schema_version must be 1") + decision = _text(raw.get("decision"), "visual plan decision", maximum=500) captures = raw.get("captures") if not isinstance(captures, list) or len(captures) > MAX_CAPTURES: raise PlanError(f"visual plan captures must be a list with at most {MAX_CAPTURES} entries") @@ -191,7 +192,11 @@ def validate_plan(raw: Any) -> dict[str, Any]: "after": _state(capture.get("after"), f"{field}.after"), } ) - return {"schema_version": 1, "captures": normalized_captures} + return { + "schema_version": 1, + "decision": decision, + "captures": normalized_captures, + } def load_plan(path: str) -> dict[str, Any]: @@ -273,9 +278,10 @@ def json_schema() -> dict[str, Any]: return { "type": "object", "additionalProperties": False, - "required": ["schema_version", "captures"], + "required": ["schema_version", "decision", "captures"], "properties": { "schema_version": {"const": 1, "type": "integer"}, + "decision": {"type": "string", "minLength": 1, "maxLength": 500}, "captures": { "type": "array", "maxItems": MAX_CAPTURES, diff --git a/scripts/release_control/release_note_visuals_test.py b/scripts/release_control/release_note_visuals_test.py index 5b49c6a26..03af382e5 100644 --- a/scripts/release_control/release_note_visuals_test.py +++ b/scripts/release_control/release_note_visuals_test.py @@ -22,6 +22,7 @@ RENDERER_SPEC.loader.exec_module(renderer) def valid_plan(): return { "schema_version": 1, + "decision": "This settings comparison makes the responsive redesign immediately visible.", "captures": [ { "id": "responsive-settings", @@ -79,6 +80,7 @@ class ReleaseNoteVisualPlanTest(unittest.TestCase): def test_structured_output_schema_carries_public_and_capture_bounds(self): schema = visuals.json_schema() self.assertEqual(schema["properties"]["schema_version"]["type"], "integer") + self.assertIn("decision", schema["required"]) captures = schema["properties"]["captures"] capture = captures["items"]["properties"] self.assertEqual(captures["maxItems"], visuals.MAX_CAPTURES) @@ -88,6 +90,18 @@ class ReleaseNoteVisualPlanTest(unittest.TestCase): visuals.MAX_STEPS, ) + def test_empty_capture_plan_requires_an_explicit_model_decision(self): + with self.assertRaisesRegex(visuals.PlanError, "decision"): + visuals.validate_plan({"schema_version": 1, "captures": []}) + plan = visuals.validate_plan( + { + "schema_version": 1, + "decision": "The investigated changes have no meaningful static visual state.", + "captures": [], + } + ) + self.assertEqual(plan["captures"], []) + def test_current_only_capture_has_one_asset(self): raw = valid_plan() raw["captures"][0]["before"] = None diff --git a/scripts/trigger-release.sh b/scripts/trigger-release.sh index 569bd7967..351dbcd74 100755 --- a/scripts/trigger-release.sh +++ b/scripts/trigger-release.sh @@ -139,13 +139,9 @@ echo "" # Check 5: Release notes file NOTES_FILE="${NOTES_FILE_ARG:-/tmp/release_notes_${VERSION}.md}" -VISUAL_PLAN_SIDECAR="${NOTES_FILE}.visuals.json" -if [ -s "$VISUAL_PLAN_SIDECAR" ]; then - VISUAL_PLAN_FILE="$VISUAL_PLAN_SIDECAR" -else - VISUAL_PLAN_FILE=$(mktemp) - rm -f "$VISUAL_PLAN_FILE" -fi +VISUAL_PLAN_FILE=$(mktemp) +rm -f "$VISUAL_PLAN_FILE" +VISUAL_PLAN_GENERATED_THIS_RUN="false" if [ -f "$NOTES_FILE" ]; then echo "Found release notes file: ${NOTES_FILE}" echo "" @@ -166,6 +162,7 @@ else echo "Generating release notes..." if RELEASE_NOTE_VISUAL_PLAN_FILE="$VISUAL_PLAN_FILE" \ ./scripts/generate-release-notes.sh "$VERSION" > "$NOTES_FILE"; then + VISUAL_PLAN_GENERATED_THIS_RUN="true" echo "Release notes generated at ${NOTES_FILE}" echo "" # Show first few lines @@ -202,8 +199,12 @@ python3 scripts/release_control/render_release_body.py \ --validate-notes-file "$NOTES_FILE" echo "✓ Release-note Markdown structure validated" -if [ ! -s "$VISUAL_PLAN_FILE" ]; then +if [ "$VISUAL_PLAN_GENERATED_THIS_RUN" != "true" ]; then + # A committed sidecar is review material, not proof that visual investigation + # ran for this dispatch. Always make the release model judge the exact notes + # and comparison range used by the publication request. ./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE" + VISUAL_PLAN_GENERATED_THIS_RUN="true" fi python3 scripts/release_control/release_note_visuals.py \ validate --plan "$VISUAL_PLAN_FILE" --output "$VISUAL_PLAN_FILE" @@ -217,7 +218,7 @@ if [ "$VISUAL_CAPTURE_COUNT" -gt 0 ]; then read -p "Use this visual plan? [Y/n] " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then - printf '%s\n' '{"schema_version":1,"captures":[]}' > "$VISUAL_PLAN_FILE" + printf '%s\n' '{"schema_version":1,"decision":"The release owner declined the model-selected visual plan.","captures":[]}' > "$VISUAL_PLAN_FILE" VISUAL_CAPTURE_COUNT=0 fi fi diff --git a/scripts/trigger-stable-patch.sh b/scripts/trigger-stable-patch.sh index 9343bba44..c4f759894 100755 --- a/scripts/trigger-stable-patch.sh +++ b/scripts/trigger-stable-patch.sh @@ -107,13 +107,8 @@ if [ ! -s "$NOTES_FILE" ]; then echo "Canonical release notes are required at ${NOTES_FILE}." >&2 exit 1 fi -VISUAL_PLAN_SIDECAR="${NOTES_FILE}.visuals.json" -if [ -s "$VISUAL_PLAN_SIDECAR" ]; then - VISUAL_PLAN_FILE="$VISUAL_PLAN_SIDECAR" -else - VISUAL_PLAN_FILE=$(mktemp) - rm -f "$VISUAL_PLAN_FILE" -fi +VISUAL_PLAN_FILE=$(mktemp) +rm -f "$VISUAL_PLAN_FILE" RESOLVER_ARGS=( --version "$VERSION" @@ -197,9 +192,10 @@ else --version "$VERSION" \ --validate-notes-file "$NOTES_FILE" - if [ ! -s "$VISUAL_PLAN_FILE" ]; then - ./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE" - fi + # A committed sidecar is review material, not proof that visual investigation + # ran for this dispatch. Always make the release model judge the exact notes + # and comparison range used by the publication request. + ./scripts/generate-release-notes.sh --visual-plan "$VERSION" "$NOTES_FILE" > "$VISUAL_PLAN_FILE" python3 scripts/release_control/release_note_visuals.py \ validate --plan "$VISUAL_PLAN_FILE" --output "$VISUAL_PLAN_FILE"