diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index dd2063129..8dbfec7c5 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -31,6 +31,7 @@ workspace = true [features] default = [] +gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"] # Compiles the controlled list-objects namespace-journal chaos injector into a # production binary (it is always available to tests). Off by default so the # RUSTFS_LIST_OBJECTS_NAMESPACE_JOURNAL_CHAOS_* env vars cannot rewrite journal @@ -212,8 +213,8 @@ aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] } parking_lot = { workspace = true } base64-simd.workspace = true serde_urlencoded.workspace = true -google-cloud-storage = { workspace = true } -google-cloud-auth = { workspace = true } +google-cloud-storage = { workspace = true, optional = true } +google-cloud-auth = { workspace = true, optional = true } faster-hex = { workspace = true } quick-xml = { workspace = true } ratelimit = { workspace = true } diff --git a/crates/ecstore/src/bucket/on_demand_migration/mod.rs b/crates/ecstore/src/bucket/on_demand_migration/mod.rs index 6147f1f94..554dadd60 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/mod.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/mod.rs @@ -30,6 +30,7 @@ mod backend_contract; pub mod backfill; pub mod breaker; pub mod config; +#[cfg(feature = "gcs")] pub mod gcs; pub mod list_through; mod native_http; diff --git a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs index 6d1ba136a..d2fc21c3e 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/source_client.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/source_client.rs @@ -26,6 +26,7 @@ //! forwarded: v1 rejects SSE-C source objects outright. use super::azure::AzureSourceBackend; +#[cfg(feature = "gcs")] use super::gcs::GcsNativeSourceBackend; use super::list_through::{ListPageError, validate_list_page}; use crate::bucket::remote_s3_client::{ @@ -722,6 +723,9 @@ impl SourceClient { )?; Ok(Self::from_backend(Box::new(backend), spec)) } + #[cfg(not(feature = "gcs"))] + SourceBackendSpec::Gcs(_) => Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")), + #[cfg(feature = "gcs")] SourceBackendSpec::Gcs(gcs) => { let backend = GcsNativeSourceBackend::new( &spec.endpoint, @@ -1113,6 +1117,28 @@ mod tests { } } + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_keeps_hmac_s3_available() { + let mut native = spec(None); + native.provider = SourceProvider::GcsNative; + native.credentials = None; + native.backend = SourceBackendSpec::Gcs(GcsSourceSpec { + service_account_json: "{}".to_string(), + }); + assert!(matches!( + SourceClient::new(&native).await, + Err(RemoteS3ClientError::BackendNotCompiled("gcs_native")) + )); + + let mut hmac = spec(None); + hmac.provider = SourceProvider::Gcs; + hmac.endpoint = "https://storage.googleapis.com".to_string(); + SourceClient::new(&hmac) + .await + .expect("GCS HMAC uses the always-available S3 backend"); + } + async fn scripted_client(spec: &SourceClientSpec, responses: Vec) -> (SourceClient, Recorded) { let requests: Recorded = Arc::new(Mutex::new(Vec::new())); let connector = SharedHttpConnector::new(ScriptedConnector { diff --git a/crates/ecstore/src/bucket/on_demand_migration/sys.rs b/crates/ecstore/src/bucket/on_demand_migration/sys.rs index 61c3043c0..ab42bd602 100644 --- a/crates/ecstore/src/bucket/on_demand_migration/sys.rs +++ b/crates/ecstore/src/bucket/on_demand_migration/sys.rs @@ -85,6 +85,8 @@ pub static GLOBAL_ON_DEMAND_MIGRATION_SYS: OnceLock = Once /// `resolve` as [`OdmLookup::Unavailable`] and through status snapshots. #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum OdmStateError { + #[error("the {0} backend is not included in this build")] + BackendNotCompiled(&'static str), /// `source.credentials` is `null`; the shared client builder has no /// anonymous mode yet (rustfs/backlog#2149 follow-up). #[error("anonymous source access is not supported yet; configure source credentials")] @@ -310,11 +312,12 @@ impl BucketOdmState { write_back: Option>, ) -> Arc { let spec = source_client_spec(config); - let client = if config.source.credentials.is_none() { + let client = if config.source.credentials.is_none() && !config.source.provider.is_native() { Err(OdmStateError::AnonymousUnsupported) } else { SourceClient::new(&spec).await.map(Arc::new).map_err(|err| match err { RemoteS3ClientError::MissingCredentials => OdmStateError::AnonymousUnsupported, + RemoteS3ClientError::BackendNotCompiled(provider) => OdmStateError::BackendNotCompiled(provider), other => OdmStateError::ClientBuild(other.to_string()), }) }; @@ -1087,6 +1090,45 @@ mod tests { assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Rebuilt); } + #[tokio::test] + async fn native_azure_uses_provider_credentials_without_s3_credentials() { + let sys = enabled_sys(); + let mut cfg = config(None); + cfg.source.provider = Provider::Azure; + cfg.source.endpoint = None; + cfg.source.credentials = None; + cfg.source.azure = Some(super::super::config::AzureSourceConfig { + account: "legacyaccount".to_string(), + account_key: Some("c2VjcmV0LWtleQ==".to_string()), + sas_token: None, + }); + assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed); + let state = ready_state(sys.resolve("b", "k")); + assert!(state.client().is_ok(), "native credentials must not be classified as anonymous S3"); + } + + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_is_unavailable_not_anonymous() { + let sys = enabled_sys(); + let mut cfg = config(None); + cfg.source.provider = Provider::GcsNative; + cfg.source.credentials = None; + cfg.source.gcs = Some(super::super::config::GcsSourceConfig { + service_account_json: "{}".to_string(), + }); + let encoded = cfg.to_json().expect("GCS config is serializable without the backend"); + let restored: OnDemandMigrationConfig = serde_json::from_slice(&encoded).expect("GCS config stays readable"); + assert_eq!(restored, cfg); + assert_eq!(sys.apply("b", Some(&cfg)).await, ApplyOutcome::Installed); + match sys.resolve("b", "k") { + Some(OdmLookup::Unavailable { error, .. }) => { + assert_eq!(error, OdmStateError::BackendNotCompiled("gcs_native")); + } + other => panic!("expected unavailable backend, got {other:?}"), + } + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn singleflight_admits_one_leader_per_key() { let sys = enabled_sys(); diff --git a/crates/ecstore/src/bucket/remote_s3_client.rs b/crates/ecstore/src/bucket/remote_s3_client.rs index a434e09ac..6389aa67e 100644 --- a/crates/ecstore/src/bucket/remote_s3_client.rs +++ b/crates/ecstore/src/bucket/remote_s3_client.rs @@ -180,6 +180,8 @@ impl RemoteS3EndpointSpec { #[derive(Debug, thiserror::Error)] pub enum RemoteS3ClientError { + #[error("the {0} backend is not included in this build")] + BackendNotCompiled(&'static str), #[error("remote endpoint requires credentials")] MissingCredentials, #[error("{0}")] diff --git a/crates/ecstore/src/services/tier/mod.rs b/crates/ecstore/src/services/tier/mod.rs index 8375f9dd0..bc22fb52a 100644 --- a/crates/ecstore/src/services/tier/mod.rs +++ b/crates/ecstore/src/services/tier/mod.rs @@ -25,6 +25,7 @@ pub(crate) mod tier_probe_intent; pub mod warm_backend; pub mod warm_backend_aliyun; pub mod warm_backend_azure; +#[cfg(feature = "gcs")] pub mod warm_backend_gcs; pub mod warm_backend_huaweicloud; pub mod warm_backend_minio; diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 22134d744..ca4864622 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -19,13 +19,14 @@ #![allow(clippy::all)] use crate::error::is_err_bucket_not_found; +#[cfg(feature = "gcs")] +use crate::services::tier::warm_backend_gcs::WarmBackendGCS; use crate::services::tier::{ tier::{ERR_TIER_BACKEND_IN_USE, ERR_TIER_INVALID_CONFIG, ERR_TIER_TYPE_UNSUPPORTED}, tier_config::{TierConfig, TierType}, tier_handlers::{ERR_TIER_BUCKET_NOT_FOUND, ERR_TIER_NOT_FOUND, ERR_TIER_PERM_ERR}, warm_backend_aliyun::WarmBackendAliyun, warm_backend_azure::WarmBackendAzure, - warm_backend_gcs::WarmBackendGCS, warm_backend_huaweicloud::WarmBackendHuaweicloud, warm_backend_minio::WarmBackendMinIO, warm_backend_r2::WarmBackendR2, @@ -912,6 +913,15 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result { + return Err(AdminError { + code: ERR_TIER_TYPE_UNSUPPORTED.code.clone(), + message: "This build does not include the GCS backend; rebuild with the gcs feature".to_string(), + status_code: StatusCode::NOT_IMPLEMENTED, + }); + } + #[cfg(feature = "gcs")] TierType::GCS => { if let Some(gcs_config) = tier.gcs.as_ref() { let dd = WarmBackendGCS::new(gcs_config, &tier.name).await; @@ -1028,6 +1038,27 @@ mod tests { const PROBE_VERSION: &str = "remote-v2"; + #[cfg(not(feature = "gcs"))] + #[tokio::test] + async fn gcs_backend_not_compiled_preserves_config() { + let json = r#"{"name":"ARCHIVE","type":"gcs","gcs":{"bucket":"archive","creds":"secret"}}"#; + let tier: TierConfig = serde_json::from_str(json).expect("GCS config remains readable without the backend"); + assert_eq!(tier.tier_type, TierType::GCS); + let encoded = serde_json::to_vec(&tier).expect("GCS config remains writable"); + let restored: TierConfig = serde_json::from_slice(&encoded).expect("GCS config round trips"); + assert_eq!(restored.tier_type, TierType::GCS); + let restored_gcs = restored.gcs.as_ref().expect("GCS settings preserved"); + assert_eq!(restored_gcs.bucket, "archive"); + assert_eq!(restored_gcs.creds, "secret"); + assert_eq!(tier.redacted().gcs.expect("redacted GCS settings").creds, "REDACTED"); + let error = match new_warm_backend(&tier, false).await { + Ok(_) => panic!("an excluded GCS backend cannot be constructed"), + Err(error) => error, + }; + assert_eq!(error.code, ERR_TIER_TYPE_UNSUPPORTED.code); + assert_eq!(error.status_code, StatusCode::NOT_IMPLEMENTED); + } + struct CountingBackend { put_result: fn() -> Result, removes: Arc, diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index d253e33a2..adf844700 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -7,6 +7,12 @@ On-Demand Migration (ODM) attaches an external S3-compatible **source bucket** t The module is on by default (rustfs/backlog#2163); set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off (`rustfs/src/module_switches.rs`). With the switch off, the runtime never intervenes on a read and the admin `PUT` route refuses with `OnDemandMigrationDisabled`. Reads of the configuration and of the status endpoint keep working while the switch is off, so a disabled deployment can still be inspected. The switch only decides whether the module may act at all: a bucket with no `on-demand-migration.json` is never resolved by the runtime and makes no source call, so turning the module on changes nothing for buckets you have not configured. +## Optional Google dependencies + +The default and `full` server builds include the `gcs` Cargo feature to preserve native GCS migration and existing GCS tier support. For a server without Google SDK dependencies, build with `cargo build -p rustfs --no-default-features --features ftps,webdav`. Add `gcs` to that feature list to restore native GCS support. The ECStore library has no default Google dependency; library users that need GCS tiers must enable its `gcs` feature. + +Both builds can read, redact and preserve GCS configuration. A build without `gcs` rejects native ODM client construction with `OnDemandMigrationBackendNotCompiled` (HTTP 501); persisted native sources report an unavailable client. GCS tier initialization returns `XRustFSAdminTierTypeUnsupported` (HTTP 501). Do not deploy that build to a cluster with GCS tiers containing transitioned objects: the configuration remains intact, but reading their remote data requires a GCS-capable binary. The `gcs` provider using HMAC credentials and the S3 interoperability API remains available in every build; only `gcs_native` and native GCS tier clients need the feature. + ## List continuation token rollout `RUSTFS_ON_DEMAND_MIGRATION_LIST_V2_TOKENS` defaults to `false`; unset or invalid boolean values also keep it off. It controls only whether a v1 listing may first issue a v2 continuation token after an empty truncated merged page. Every node with this reader support accepts existing v2 tokens and continues their budget even with the switch off. Ordinary pages that consume an object or common prefix retain the original v1 token shape. diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 6d716d63b..063ce400e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -57,7 +57,8 @@ name = "swift_object_integration_test" required-features = ["swift"] [features] -default = ["ftps", "webdav"] +default = ["ftps", "webdav", "gcs"] +gcs = ["rustfs-ecstore/gcs"] metrics-gpu = ["rustfs-obs/gpu"] ftps = ["rustfs-protocols/ftps"] swift = ["rustfs-protocols/swift"] @@ -66,7 +67,7 @@ sftp = ["rustfs-protocols/sftp"] license = [] io-scheduler-debug = [] # Enable debug information in I/O scheduler tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only) -full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"] +full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"] e2e-test-hooks = [] # Shortens Connect credentials only in debug E2E builds. connect-e2e-short-credentials = [] diff --git a/rustfs/src/admin/handlers/on_demand_migration.rs b/rustfs/src/admin/handlers/on_demand_migration.rs index 103e6ee8d..d8bc9d713 100644 --- a/rustfs/src/admin/handlers/on_demand_migration.rs +++ b/rustfs/src/admin/handlers/on_demand_migration.rs @@ -90,6 +90,8 @@ const BACKFILL_OP_CANCEL: &str = "cancel"; pub(crate) const ERR_CODE_MODULE_DISABLED: &str = "OnDemandMigrationDisabled"; /// Error code returned when the source bucket did not answer the probe. pub(crate) const ERR_CODE_SOURCE_UNREACHABLE: &str = "OnDemandMigrationSourceUnreachable"; +/// Error code returned when the configured provider was excluded at build time. +pub(crate) const ERR_CODE_BACKEND_NOT_COMPILED: &str = "OnDemandMigrationBackendNotCompiled"; /// Error code returned by `GET` when the bucket has no configuration. pub(crate) const ERR_CODE_NO_SUCH_CONFIGURATION: &str = "NoSuchConfiguration"; /// Error code (409) returned by `start` while a backfill job holds the lease. @@ -630,12 +632,17 @@ pub(crate) fn source_client_spec(config: &OnDemandMigrationConfig) -> SourceClie } } -/// Builder failures are input errors: the endpoint policy, the CA PEM or the -/// credentials the operator supplied. Anonymous sources are not wired yet +/// Distinguishes excluded backends from invalid endpoint, CA or credentials. +/// Anonymous S3 sources are not wired yet /// (ODM-05 adds the credential-less path), so `MissingCredentials` is a 400 /// naming the field instead of an opaque internal error. fn client_build_error(err: RemoteS3ClientError) -> S3Error { match err { + RemoteS3ClientError::BackendNotCompiled(provider) => custom_error( + ERR_CODE_BACKEND_NOT_COMPILED, + StatusCode::NOT_IMPLEMENTED, + format!("the {provider} backend is not included in this build; rebuild with the gcs feature"), + ), RemoteS3ClientError::MissingCredentials => admin_s3_error( S3ErrorCode::InvalidArgument, "source.credentials is required: anonymous sources are not supported yet", @@ -1288,6 +1295,14 @@ mod tests { assert!(err.message().unwrap_or_default().contains("source.credentials")); } + #[test] + fn backend_not_compiled_is_distinct_from_invalid_credentials() { + let err = client_build_error(RemoteS3ClientError::BackendNotCompiled("gcs_native")); + assert_eq!(err.code(), &S3ErrorCode::Custom(ERR_CODE_BACKEND_NOT_COMPILED.into())); + assert_eq!(err.status_code(), Some(StatusCode::NOT_IMPLEMENTED)); + assert!(err.message().unwrap_or_default().contains("gcs feature")); + } + #[test] fn module_switch_defaults_on_and_reads_the_env() { temp_env::with_var(ENV_ON_DEMAND_MIGRATION_ENABLED, None::<&str>, || assert!(module_enabled())); diff --git a/rustfs/src/app/object/shared.rs b/rustfs/src/app/object/shared.rs index 454cd9911..8f0808956 100644 --- a/rustfs/src/app/object/shared.rs +++ b/rustfs/src/app/object/shared.rs @@ -992,7 +992,7 @@ pub(crate) fn odm_source_error_response(policy: &PolicyConfig, class: &'static s /// Metrics/message label for a bucket whose source client could not be built. pub(crate) fn odm_state_error_class(error: &OdmStateError) -> &'static str { match error { - OdmStateError::AnonymousUnsupported => "unsupported", + OdmStateError::AnonymousUnsupported | OdmStateError::BackendNotCompiled(_) => "unsupported", OdmStateError::ClientBuild(_) => "client_build", } } @@ -2035,6 +2035,7 @@ mod on_demand_migration_tests { #[test] fn odm_state_error_class_is_stable() { assert_eq!(odm_state_error_class(&OdmStateError::AnonymousUnsupported), "unsupported"); + assert_eq!(odm_state_error_class(&OdmStateError::BackendNotCompiled("gcs_native")), "unsupported"); assert_eq!(odm_state_error_class(&OdmStateError::ClientBuild("tls".to_string())), "client_build"); }