feat(ecstore): add native azure blob and gcs migration sources (#7211)

* feat(ecstore): add a native azure blob odm source backend

* feat(ecstore): add a native gcs odm source backend and one backend contract

* fix(ecstore): refuse an empty azure account key at client build

* fix(ecstore): probe gcs sources with the listing permission

* fix(app): drop a redundant match guard on the sse config lookup

* fix(ecstore): drop stale rename commit duplicates from local.rs

* test(ecstore): use the sanctioned placeholder key in the gcs fixture

rustfs/src/admin/handlers/on_demand_migration.rs

@@ -46,6 +46,7 @@ use crate::admin::storage_api::bucket::on_demand_migration::source_client::{
 };
 use crate::admin::storage_api::bucket::on_demand_migration::{
     OdmBucketSnapshot, OnDemandMigrationConfig, OnDemandMigrationConfigError, OnDemandMigrationSys, PathStyle, ValidationContext,
+    source_backend_spec,
 };
 use crate::admin::storage_api::bucket::remote_s3_client::{
     PathStyle as RemotePathStyle, RemoteCredentials, RemoteS3ClientError, RemoteS3RetryPolicy,
@@ -585,6 +586,8 @@ fn source_provider(config: &OnDemandMigrationConfig) -> SourceProvider {
         Provider::Rustfs => SourceProvider::Rustfs,
         Provider::R2 => SourceProvider::R2,
         Provider::Gcs => SourceProvider::Gcs,
+        Provider::Azure => SourceProvider::Azure,
+        Provider::GcsNative => SourceProvider::GcsNative,
     }
 }
 
@@ -621,6 +624,9 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie
         // a flapping source behind a success and triple the probe's cost.
         retry: RemoteS3RetryPolicy::Disabled,
         bandwidth_limit: config.policy.bandwidth_limit_bytes_per_sec.and_then(NonZeroU64::new),
+        // One mapping serves the probe and the runtime, so an admin probe
+        // always exercises the backend the runtime will build.
+        backend: source_backend_spec(source),
     }
 }
 

rustfs/src/admin/storage_api.rs

@@ -292,6 +292,7 @@ pub(crate) mod on_demand_migration {
     pub(crate) type PathStyle = super::ecstore_bucket::on_demand_migration::PathStyle;
     pub(crate) type Provider = super::ecstore_bucket::on_demand_migration::Provider;
     pub(crate) type ValidationContext<'a> = super::ecstore_bucket::on_demand_migration::ValidationContext<'a>;
+    pub(crate) use super::ecstore_bucket::on_demand_migration::source_backend_spec;
 
     pub(crate) mod backfill {
         pub(crate) type BackfillCheckpoint = super::super::ecstore_bucket::on_demand_migration::backfill::BackfillCheckpoint;

rustfs/src/app/object/get.rs

@@ -4832,6 +4832,8 @@ mod on_demand_migration_tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: None,

rustfs/src/app/object/head.rs

@@ -665,6 +665,8 @@ mod tests {
                     session_token: None,
                 }),
                 tls: TlsConfig::default(),
+                azure: None,
+                gcs: None,
             },
             filter: FilterConfig {
                 prefix: None,
@@ -743,6 +745,7 @@ mod tests {
                 },
             ),
             is_multipart_etag: true,
+            etag_is_opaque: false,
         }
     }
 

rustfs/src/app/object/on_demand_migration_put.rs

@@ -124,6 +124,12 @@ pub(super) fn expected_md5_hex(head: &SourceHead) -> Option<String> {
     if head.sse.is_some() {
         return None;
     }
+    // Azure stamps an opaque concurrency token in the ETag slot. It is
+    // recorded as provenance, but reading it as a digest would compare the
+    // pulled bytes against a value that never described them.
+    if head.etag_is_opaque {
+        return None;
+    }
     let etag = head.etag.as_deref()?;
     if etag.len() != 32 || is_multipart_etag(etag) || !etag.bytes().all(|byte| byte.is_ascii_hexdigit()) {
         return None;
@@ -1043,6 +1049,12 @@ mod tests {
         head.sse = None;
         head.etag = None;
         assert_eq!(expected_md5_hex(&head), None);
+
+        // An Azure ETag can be any string the service chooses; even one that
+        // happens to look like an MD5 must not be checked against the bytes.
+        let mut head = source_head(b"abc");
+        head.etag_is_opaque = true;
+        assert_eq!(expected_md5_hex(&head), None, "opaque provider ETag");
     }
 
     #[test]