test(odm): verify rc5 rollback configuration recovery (#7224)

* test(odm): verify rc5 rollback configuration recovery

* ci(e2e): run the ODM rollback recovery scenario

.github/workflows/e2e-upgrade.yml

@@ -78,6 +78,10 @@ jobs:
             cache_key: e2e-bucket-config-rollback
             test: rollback_to_previous_release_reads_current_bucket_metadata
             artifact: bucket-config-rollback
+          - name: ODM configuration recovery after rc.5 rollback
+            cache_key: e2e-odm-config-rollback
+            test: rc5_rollback_requires_restoring_odm_configuration
+            artifact: odm-config-rollback
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:

crates/e2e_test/src/upgrade_compatibility_test.rs

@@ -15,7 +15,8 @@
 use crate::common::{
     RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging, replication_fast_env, rustfs_binary_path,
 };
-use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
+use crate::on_demand_migration::common::{ODM_SERVER_ENV, OdmTestEnv, SeedObject};
 use crate::replication_extension_test::{
     LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, put_bucket_replication, set_replication_target_with_options,
 };
@@ -38,6 +39,7 @@ type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
 type BoxError = Box<dyn std::error::Error + Send + Sync>;
 
 const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
+const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
 const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
 const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
 const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -277,6 +279,93 @@ async fn exercise_mixed_cluster(
     Ok(())
 }
 
+/// Pins the published old writer's limitation and the supported recovery
+/// procedure. This is not a promise that mixed-version ODM is supported.
+/// Replace the loss assertion when ODM gains independent persistence;
+/// preserving configuration across rc.5 writes is then an improvement.
+#[tokio::test]
+#[ignore = "requires the pinned 1.0.0-rc.5 release binary"]
+async fn rc5_rollback_requires_restoring_odm_configuration() -> TestResult {
+    init_logging();
+    let previous_binary = source_binary()?;
+    let version = tokio::process::Command::new(&previous_binary)
+        .arg("--version")
+        .output()
+        .await?;
+    assert!(version.status.success(), "previous binary must report its version");
+    assert!(
+        String::from_utf8(version.stdout)?.contains(RC5_COMMIT),
+        "this compatibility scenario requires the published rc.5 writer"
+    );
+    let mut env = OdmTestEnv::start().await?;
+    let bucket = "odm-rc5-rollback";
+    let source_bucket = "odm-rc5-source";
+    env.source.create_bucket_with_mode(source_bucket, BucketMode::Unversioned);
+    env.seed_source(
+        source_bucket,
+        &[SeedObject::new(
+            "source-only",
+            bytes::Bytes::from_static(b"source read after recovery"),
+        )],
+    );
+    env.rustfs.create_test_bucket(bucket).await?;
+    let saved_config = env.fake_source_spec(source_bucket);
+    assert_eq!(env.configure_source(bucket, &saved_config).await?.status, 200);
+    let before = env.get_config(bucket).await?;
+    assert_eq!(before.status, 200);
+    let expected_config = before
+        .json()?
+        .get("config")
+        .cloned()
+        .ok_or("configuration response omitted config")?;
+    env.client
+        .put_object()
+        .bucket(bucket)
+        .key("local")
+        .body(ByteStream::from_static(b"local data survives rollback"))
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restarted = env.get_config(bucket).await?;
+    assert_eq!(restarted.status, 200, "a current writer preserves ODM across restart");
+    assert_eq!(restarted.json()?.get("config"), Some(&expected_config));
+
+    restart_from_binary(&mut env.rustfs, &previous_binary, &[]).await?;
+    env.client
+        .put_bucket_tagging()
+        .bucket(bucket)
+        .tagging(
+            Tagging::builder()
+                .tag_set(Tag::builder().key("writer").value("rc5").build()?)
+                .build()?,
+        )
+        .send()
+        .await?;
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let missing = env.get_config(bucket).await?;
+    assert_eq!(missing.status, 404, "rc.5 rewrites metadata without ODM keys");
+    assert!(missing.body.contains("NoSuchConfiguration"));
+    assert_eq!(read_object(&env.client, bucket, "local", None).await?.1, b"local data survives rollback");
+    let tags = env.client.get_bucket_tagging().bucket(bucket).send().await?;
+    assert!(tags.tag_set().iter().any(|tag| tag.key() == "writer" && tag.value() == "rc5"));
+
+    assert_eq!(
+        env.configure_source(bucket, &saved_config).await?.status,
+        200,
+        "restore from saved full configuration"
+    );
+    env.rustfs.restart_server_preserving_data(vec![], ODM_SERVER_ENV).await?;
+    let restored = env.get_config(bucket).await?;
+    assert_eq!(restored.status, 200, "restored ODM configuration persists");
+    assert_eq!(restored.json()?.get("config"), Some(&expected_config));
+    env.wait_until_source_consulted(bucket).await?;
+    assert_eq!(
+        read_object(&env.client, bucket, "source-only", None).await?.1,
+        b"source read after recovery"
+    );
+    Ok(())
+}
+
 #[tokio::test]
 #[ignore = "requires a pinned previous RustFS release binary"]
 async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult {

docs/operations/on-demand-migration.md

@@ -7,6 +7,14 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t
 
 The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured.
 
+## Upgrade and rollback compatibility
+
+ODM configuration is stored in two additional keys in the existing bucket metadata map. The metadata format version remains `1` for MinIO compatibility. RustFS `1.0.0-rc.5` only re-encodes its 44 known keys: a bucket configuration write through an rc.5 node discards the ODM configuration and timestamp, even if another node originally wrote them. Restarting a newer binary cannot recover the discarded values. This also means ODM is not supported during a rolling upgrade that still allows rc.5 nodes to write bucket metadata.
+
+Upgrade every node before enabling ODM. Before any rollback to rc.5, stop new migration work, retain a secure copy of the original full configuration and credentials, and disable ODM on every bucket and node. The redacted configuration GET and metadata export are not credential backups. Objects still present only at the source cannot be read through RustFS while ODM is disabled or rc.5 is running; finish migration first, redirect those reads to the source, or plan a maintenance window. After all nodes return to a compatible release, reapply and validate the saved configuration; already stored local objects remain local. Turning the global module switch off alone does not make an old metadata writer preserve these keys.
+
+The ignored `upgrade_compatibility_test::rc5_rollback_requires_restoring_odm_configuration` test pins release commit `40a2470feb567201165a5b809b7598bb4b1f68f5`, restarts against the same data directory, writes bucket tags through rc.5, and verifies configuration recovery after returning to the current binary. Set `RUSTFS_UPGRADE_SOURCE_BINARY` to that release's executable and run `cargo test -p e2e_test rc5_rollback_requires_restoring_odm_configuration -- --ignored --test-threads=1`. The test records a known old-writer limitation; it does not certify mixed-version ODM operation.
+
 ## Optional Google dependencies
 
 The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature.