test(e2e): emit scanner heal G09 upgrade evidence (#7519)

Record measured Scanner/Heal G09 evidence artifacts from the upgrade compatibility lanes when a fresh evidence directory is provided.

Co-authored-by: zhi22915 <qiuzgang@gmail.com>

.github/workflows/e2e-upgrade.yml

@@ -129,11 +129,22 @@ jobs:
           : > target/debug/rustfs.features
 
       - name: Run upgrade compatibility test
+        env:
+          RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
         run: |
           cargo test --locked -p e2e_test \
             "upgrade_compatibility_test::${{ matrix.test }}" \
             -- --ignored --exact --nocapture
 
+      - name: Upload scanner/heal G09 evidence
+        if: always()
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
+        with:
+          name: ${{ matrix.artifact }}-g09-evidence-${{ github.run_number }}
+          path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence/${{ matrix.artifact }}
+          if-no-files-found: ignore
+          retention-days: 14
+
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -32,8 +32,10 @@ use aws_sdk_s3::types::{
     VersioningConfiguration,
 };
 use http::{Method, StatusCode};
+use serde_json::Value;
+use std::io::Write;
 use std::path::{Path, PathBuf};
-use std::time::Duration;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 use tokio::task::JoinSet;
 use tokio::time::{Instant, sleep};
 
@@ -41,6 +43,7 @@ type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
 const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
@@ -81,6 +84,14 @@ const QUOTA_READINESS_TIMEOUT: Duration = Duration::from_secs(30);
 // of treating it as an upgrade failure.
 const QUOTA_ADMISSION_WARMUP_TIMEOUT: Duration = Duration::from_secs(90);
 
+struct G09EvidenceContext {
+    directory: PathBuf,
+    current_revision: String,
+    previous_revision: String,
+    run_id: String,
+    measurement_window_id: String,
+}
+
 fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> {
     let path = std::env::var_os(SOURCE_BINARY_ENV)
         .map(PathBuf::from)
@@ -91,6 +102,155 @@ fn source_binary() -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>>
     Ok(path)
 }
 
+fn is_lower_hex_revision(value: &str) -> bool {
+    value.len() == 40
+        && value
+            .bytes()
+            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
+}
+
+fn current_source_revision() -> Result<String, BoxError> {
+    let revision = env!("RUSTFS_E2E_BUILD_COMMIT");
+    if !is_lower_hex_revision(revision) {
+        return Err(format!("current test binary has invalid source revision {revision}").into());
+    }
+    Ok(revision.to_string())
+}
+
+async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
+    let output = tokio::process::Command::new(binary).arg("--version").output().await?;
+    if !output.status.success() {
+        return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
+    }
+    let version = String::from_utf8(output.stdout)?;
+    source_revision_from_version_output(&version, binary)
+}
+
+fn source_revision_from_version_output(version: &str, binary: &Path) -> Result<String, BoxError> {
+    version
+        .split(|ch: char| !ch.is_ascii_hexdigit())
+        .find(|token| is_lower_hex_revision(token))
+        .map(str::to_string)
+        .ok_or_else(|| format!("{} --version did not expose a 40-byte source revision", binary.display()).into())
+}
+
+async fn g09_evidence_context(previous_binary: &Path) -> Result<Option<G09EvidenceContext>, BoxError> {
+    let Some(directory) = std::env::var_os(G09_EVIDENCE_DIR_ENV) else {
+        return Ok(None);
+    };
+    let directory = PathBuf::from(directory);
+    std::fs::create_dir_all(&directory)?;
+    let current_revision = current_source_revision()?;
+    let previous_revision = binary_source_revision(previous_binary).await?;
+    if current_revision == previous_revision {
+        return Err("G09 mixed-version evidence requires distinct current and previous source revisions".into());
+    }
+    let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
+    Ok(Some(G09EvidenceContext {
+        directory,
+        current_revision,
+        previous_revision,
+        run_id: format!("g09-upgrade-{}-{now}", std::process::id()),
+        measurement_window_id: format!("g09-mixed-version-window-{now}"),
+    }))
+}
+
+fn write_g09_evidence(
+    context: &G09EvidenceContext,
+    field: &str,
+    role: &str,
+    cases: &[&str],
+    test: &str,
+    details: Value,
+) -> TestResult {
+    let path = context.directory.join(format!("G09-{field}.json"));
+    let mut evidence = serde_json::json!({
+        "schema": 1,
+        "evidence_type": "measured",
+        "artifact_kind": "upgrade-compatibility-e2e",
+        "source_revision": context.current_revision,
+        "run_id": context.run_id,
+        "measurement_window_id": context.measurement_window_id,
+        "gate": "G09",
+        "field": field,
+        "versions": [context.previous_revision, context.current_revision],
+        "current_revision": context.current_revision,
+        "previous_revision": context.previous_revision,
+        "mixed_version_role": role,
+        "mixed_version_cases": cases,
+        "test": test,
+        "details": details,
+    });
+    if field == "rollback_payload_evidence" {
+        evidence["rollback_payload_replayed"] = Value::Bool(true);
+    }
+    let data = serde_json::to_vec_pretty(&evidence)?;
+    if data.len() > 1024 * 1024 {
+        return Err("G09 mixed-version evidence exceeds the 1 MiB artifact budget".into());
+    }
+    let mut output = std::fs::OpenOptions::new().write(true).create_new(true).open(path)?;
+    output.write_all(&data)?;
+    output.write_all(b"\n")?;
+    output.sync_all()?;
+    Ok(())
+}
+
+#[cfg(test)]
+mod g09_evidence_tests {
+    use super::*;
+    use tempfile::TempDir;
+
+    #[test]
+    fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
+        let binary = Path::new("rustfs");
+        assert_eq!(
+            source_revision_from_version_output("rustfs 1.0.0 abcdef0123456789abcdef0123456789abcdef01 clean", binary)?,
+            "abcdef0123456789abcdef0123456789abcdef01"
+        );
+        assert!(source_revision_from_version_output("rustfs ABCDEF0123456789ABCDEF0123456789ABCDEF01", binary).is_err());
+        assert!(source_revision_from_version_output("rustfs abcdef", binary).is_err());
+        Ok(())
+    }
+
+    #[test]
+    fn g09_evidence_writer_records_measured_role_and_refuses_overwrite() -> TestResult {
+        let directory = TempDir::new()?;
+        let context = G09EvidenceContext {
+            directory: directory.path().to_path_buf(),
+            current_revision: "b".repeat(40),
+            previous_revision: "a".repeat(40),
+            run_id: "g09-upgrade-test-run".to_string(),
+            measurement_window_id: "g09-upgrade-test-window".to_string(),
+        };
+        write_g09_evidence(
+            &context,
+            "mixed_version_reader_evidence",
+            "mixed-version-reader",
+            &["old-writer-new-reader", "new-writer-old-reader"],
+            "unit",
+            serde_json::json!({"assertions": ["reader evidence"]}),
+        )?;
+        let path = directory.path().join("G09-mixed_version_reader_evidence.json");
+        let evidence: Value = serde_json::from_slice(&std::fs::read(path)?)?;
+        assert_eq!(evidence["evidence_type"], "measured");
+        assert_eq!(evidence["gate"], "G09");
+        assert_eq!(evidence["field"], "mixed_version_reader_evidence");
+        assert_eq!(evidence["mixed_version_role"], "mixed-version-reader");
+        assert_eq!(evidence["versions"], serde_json::json!(["a".repeat(40), "b".repeat(40)]));
+
+        let overwrite = write_g09_evidence(
+            &context,
+            "mixed_version_reader_evidence",
+            "mixed-version-reader",
+            &["old-writer-new-reader", "new-writer-old-reader"],
+            "unit",
+            serde_json::json!({}),
+        );
+        assert!(overwrite.is_err(), "G09 evidence must not overwrite an existing artifact");
+        Ok(())
+    }
+}
+
 async fn enable_versioning(client: &Client, bucket: &str) -> TestResult {
     let configuration = VersioningConfiguration::builder()
         .status(BucketVersioningStatus::Enabled)
@@ -523,6 +683,7 @@ async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {
 async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    let evidence_context = g09_evidence_context(&previous_binary).await?;
     let current_binary = rustfs_binary_path();
     let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
     cluster.set_env("RUST_LOG", "rustfs=warn,rustfs_notify=warn");
@@ -555,6 +716,45 @@ async fn rolling_upgrade_from_rc2_preserves_mixed_version_contracts() -> TestRes
         }
     }
 
+    if let Some(context) = evidence_context.as_ref() {
+        let phases = ["one-current-node", "one-previous-node"];
+        let objects_per_phase = MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER + 2;
+        write_g09_evidence(
+            context,
+            "mixed_version_reader_evidence",
+            "mixed-version-reader",
+            &["old-writer-new-reader", "new-writer-old-reader"],
+            "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts",
+            serde_json::json!({
+                "bucket": MIXED_BUCKET,
+                "phases": phases,
+                "objects_per_phase": objects_per_phase,
+                "assertions": [
+                    "current node reads objects written through previous-release client",
+                    "previous-release node reads objects written through current client",
+                    "all nodes list every mixed-version object after homogeneous-current convergence"
+                ],
+            }),
+        )?;
+        write_g09_evidence(
+            context,
+            "mixed_version_writer_evidence",
+            "mixed-version-writer",
+            &["old-reader-new-writer", "new-reader-old-writer"],
+            "upgrade_compatibility_test::rolling_upgrade_from_rc2_preserves_mixed_version_contracts",
+            serde_json::json!({
+                "bucket": MIXED_BUCKET,
+                "phases": phases,
+                "objects_per_phase": objects_per_phase,
+                "assertions": [
+                    "current writer publishes objects readable by previous-release node",
+                    "previous-release writer publishes objects readable by current node",
+                    "multipart writers continue under one-current-node and one-previous-node layouts"
+                ],
+            }),
+        )?;
+    }
+
     Ok(())
 }
 
@@ -1136,6 +1336,7 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
 async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
     init_logging();
     let previous_binary = source_binary()?;
+    let evidence_context = g09_evidence_context(&previous_binary).await?;
 
     let replication_target = FakeS3Target::start().await?;
     replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
@@ -1204,6 +1405,54 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
     );
     assert_eq!(body, post_rollback_bytes);
 
+    env.restart_server_preserving_data(vec![], &server_env).await?;
+    let current_again = env.create_s3_client();
+    assert_versioning_enabled(&current_again, ROLLBACK_BUCKET, "after rolling forward again").await?;
+    assert_default_sse_s3_encryption(&current_again, ROLLBACK_BUCKET, "after rolling forward again").await?;
+    assert_bucket_tag(&current_again, ROLLBACK_BUCKET, "after rolling forward again").await?;
+    assert_remote_target_preserved(&env, ROLLBACK_BUCKET, &target_arn, "after rolling forward again").await?;
+    assert_eq!(
+        read_object(&current_again, ROLLBACK_BUCKET, single_key, Some(&single_version))
+            .await?
+            .1,
+        single_bytes
+    );
+    assert_eq!(
+        read_object(&current_again, ROLLBACK_BUCKET, multipart_key, None).await?.1,
+        multipart_bytes
+    );
+    assert_eq!(
+        read_object(&current_again, ROLLBACK_BUCKET, post_rollback_key, None).await?.1,
+        post_rollback_bytes
+    );
+
+    if let Some(context) = evidence_context.as_ref() {
+        write_g09_evidence(
+            context,
+            "rollback_payload_evidence",
+            "rollback-payload",
+            &["rollback-to-old", "rollback-to-new", "unknown-field-retained"],
+            "upgrade_compatibility_test::rollback_to_previous_release_reads_current_bucket_metadata",
+            serde_json::json!({
+                "bucket": ROLLBACK_BUCKET,
+                "cases": {
+                    "rollback-to-old": [
+                        "previous-release binary reads current-build versioning, SSE-S3, tags, replication target and objects",
+                        "previous-release writer honors the decoded current-build encryption configuration"
+                    ],
+                    "rollback-to-new": [
+                        "current build reads the object written by the rolled-back previous release",
+                        "current build reads the current-build single-part and multipart objects after rolling forward again"
+                    ],
+                    "unknown-field-retained": [
+                        "previous release skips current-build bucket metadata extension fields without dropping known bucket configuration",
+                        "current build reads the retained bucket configuration after the previous-release round trip"
+                    ]
+                },
+            }),
+        )?;
+    }
+
     replication_target.shutdown().await;
     Ok(())
 }

docs/testing/ci-gates.md

@@ -262,6 +262,18 @@ mixed-peer fallback oracles; G09 keeps mixed-version reader, writer, and rollbac
 payload evidence explicit. These fields are part of the release contract, not
 evidence by themselves.
 
+The upgrade compatibility E2E can emit raw G09 JSON artifacts when
+`RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR` points at a fresh, task-owned directory.
+The rolling mixed-version test writes `G09-mixed_version_reader_evidence.json`
+and `G09-mixed_version_writer_evidence.json` after the old/new reader and writer
+assertions pass. The bucket-metadata rollback test writes
+`G09-rollback_payload_evidence.json` after the current -> previous -> current
+round trip has read back the known bucket configuration and objects. These
+artifacts are measured inputs for a later release bundle; the bundle must still
+record their relative paths, hashes, command provenance, timestamps, roles,
+participating revisions, and case lists before
+`--check-scanner-heal-release-bundle` can validate them.
+
 When the real release lanes have produced their dedicated artifacts, validate
 the complete hard-gate bundle with: