mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 16:28:15 +00:00
refactor: prune notify ecstore object alias (#3620)
This commit is contained in:
Generated
+1
@@ -9019,6 +9019,7 @@ dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"chacha20poly1305",
|
||||
"chrono",
|
||||
"clap",
|
||||
"const-str",
|
||||
"datafusion",
|
||||
|
||||
@@ -17,11 +17,11 @@ use std::sync::Arc;
|
||||
pub(crate) const IAM_CONFIG_ROOT_PREFIX: &str = rustfs_ecstore::config::RUSTFS_CONFIG_PREFIX;
|
||||
|
||||
pub(crate) type IamEcstoreError = rustfs_ecstore::error::Error;
|
||||
pub(crate) type IamConfigObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
pub(crate) type IamStorageError = rustfs_ecstore::error::StorageError;
|
||||
pub(crate) type IamStorageResult<T> = rustfs_ecstore::error::Result<T>;
|
||||
pub(crate) type IamStore = rustfs_ecstore::store::ECStore;
|
||||
pub(crate) type IamConfigObjectInfo = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub(crate) type IamConfigObjectOptions = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
|
||||
pub(crate) async fn read_iam_config_no_lock(api: Arc<IamStore>, file: &str) -> IamStorageResult<Vec<u8>> {
|
||||
rustfs_ecstore::config::com::read_config_no_lock(api, file).await
|
||||
|
||||
@@ -12,5 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(super) type IamObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
pub(super) type IamObjectOptions = rustfs_ecstore::object_api::ObjectOptions;
|
||||
use crate::storage_compat::IamStore;
|
||||
|
||||
pub(super) type IamObjectInfo = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectInfo;
|
||||
pub(super) type IamObjectOptions = <IamStore as rustfs_storage_api::ObjectOperations>::ObjectOptions;
|
||||
|
||||
@@ -12,14 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_config::server_config::Config;
|
||||
use rustfs_ecstore::{config, global};
|
||||
|
||||
use crate::event::NotifyObjectInfo;
|
||||
|
||||
type EcstoreObjectInfo = rustfs_ecstore::object_api::ObjectInfo;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum NotifyConfigStoreError {
|
||||
StorageNotAvailable,
|
||||
@@ -49,73 +44,3 @@ where
|
||||
|
||||
Ok(Some(new_config))
|
||||
}
|
||||
|
||||
impl From<EcstoreObjectInfo> for NotifyObjectInfo {
|
||||
fn from(object: EcstoreObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: object.bucket,
|
||||
name: object.name,
|
||||
size: object.size,
|
||||
etag: object.etag,
|
||||
content_type: object.content_type,
|
||||
user_defined: object
|
||||
.user_defined
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect(),
|
||||
version_id: object.version_id.map(|version_id| version_id.to_string()),
|
||||
mod_time: object
|
||||
.mod_time
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value.unix_timestamp(), value.nanosecond())),
|
||||
restore_expires: object
|
||||
.restore_expires
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value.unix_timestamp(), value.nanosecond())),
|
||||
storage_class: object.storage_class,
|
||||
transitioned_tier: (!object.transitioned_object.tier.is_empty()).then_some(object.transitioned_object.tier),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_storage_api::TransitionedObject;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
|
||||
#[test]
|
||||
fn ecstore_object_info_conversion_preserves_notify_event_fields() {
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + Duration::seconds(42);
|
||||
let restore_expires = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1_700_000_000);
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("x-amz-meta-key".to_string(), "value".to_string());
|
||||
|
||||
let converted = NotifyObjectInfo::from(EcstoreObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 123,
|
||||
etag: Some("etag".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_defined: Arc::new(metadata),
|
||||
mod_time: Some(mod_time),
|
||||
restore_expires: Some(restore_expires),
|
||||
storage_class: Some("GLACIER".to_string()),
|
||||
transitioned_object: TransitionedObject {
|
||||
tier: "DEEP_ARCHIVE".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(converted.bucket, "bucket");
|
||||
assert_eq!(converted.name, "object");
|
||||
assert_eq!(converted.size, 123);
|
||||
assert_eq!(converted.etag.as_deref(), Some("etag"));
|
||||
assert_eq!(converted.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(converted.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
|
||||
assert_eq!(converted.mod_time, DateTime::<Utc>::from_timestamp(42, 0));
|
||||
assert_eq!(converted.restore_expires, DateTime::<Utc>::from_timestamp(1_700_000_000, 0));
|
||||
assert_eq!(converted.storage_class.as_deref(), Some("GLACIER"));
|
||||
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,16 +5,16 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
## Current Context
|
||||
|
||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||
- Branch: `overtrue/arch-ecstore-object-boundary-guards`
|
||||
- Baseline: latest `main` after `rustfs/rustfs#3616`
|
||||
(`8d91e2116f53fabc5f8a12a7cc48eb9d72cfcf10`).
|
||||
- Branch: `overtrue/arch-iam-object-boundary-prune`
|
||||
- Baseline: stacked after `rustfs/rustfs#3620`
|
||||
(`c7bee5de904c9a1ea70f73ad6621895a30b3c387`).
|
||||
- PR type for this branch: `consumer-migration`
|
||||
- Runtime behavior changes: none.
|
||||
- Rust code changes: none.
|
||||
- CI/script changes: snapshot the exact remaining external
|
||||
`rustfs_ecstore::object_api` compatibility aliases and reject new
|
||||
object-api names outside the approved boundary surface.
|
||||
- Docs changes: record the API-067 ECStore object API boundary guard slice.
|
||||
- Rust code changes: route IAM config/store object metadata and options aliases
|
||||
through `rustfs_storage_api::ObjectOperations` associated types.
|
||||
- CI/script changes: shrink the remaining external `rustfs_ecstore::object_api`
|
||||
compatibility alias snapshot by removing IAM object-info/options aliases.
|
||||
- Docs changes: record the API-069 IAM object boundary prune slice.
|
||||
|
||||
## Phase 0 Tasks
|
||||
|
||||
@@ -331,6 +331,35 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
||||
- Verification: bash syntax check, migration and layer guards, formatting,
|
||||
diff hygiene, full pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-068` Prune notify ECStore object-info compatibility alias.
|
||||
- Completed slice: remove notify's private `EcstoreObjectInfo` alias and
|
||||
ECStore-object conversion implementation, then map ECStore event objects to
|
||||
`NotifyObjectInfo` inside the RustFS event and operation notification
|
||||
bridges.
|
||||
- Acceptance: `crates/notify` no longer references
|
||||
`rustfs_ecstore::object_api::ObjectInfo`, the remaining object-api alias
|
||||
allowlist shrinks accordingly, and notify event payload fields keep the
|
||||
same serialized values.
|
||||
- Must preserve: live event dispatch behavior, event names, bucket/object
|
||||
fields, version IDs, metadata, restore-completed timestamps, storage class,
|
||||
transitioned tier, host/port parsing, and replication request filtering.
|
||||
- Verification: focused RustFS event conversion test, focused notify/RustFS
|
||||
compile checks, migration and layer guards, formatting, diff hygiene, full
|
||||
pre-commit, and three-expert review.
|
||||
|
||||
- [x] `API-069` Prune IAM direct ECStore object metadata/options aliases.
|
||||
- Completed slice: replace IAM config and store `ObjectInfo`/`ObjectOptions`
|
||||
compatibility aliases with `IamStore` `ObjectOperations` associated types.
|
||||
- Acceptance: IAM no longer names
|
||||
`rustfs_ecstore::object_api::{ObjectInfo,ObjectOptions}` directly, the
|
||||
remaining object-api alias allowlist shrinks by four entries, and IAM config
|
||||
read/write metadata and lazy-rewrite precondition behavior are unchanged.
|
||||
- Must preserve: IAM config encryption/decryption, lazy rewrite ETag matching,
|
||||
list walk item/error typing, metadata return shape, storage preconditions,
|
||||
system-path failure classification, and notification peer behavior.
|
||||
- Verification: focused IAM compile/tests, migration and layer guards,
|
||||
formatting, diff hygiene, full pre-commit, and three-expert review.
|
||||
|
||||
- [x] `TEST-PRTYPE-001` Check PR type enum consistency.
|
||||
- Acceptance: `./scripts/check_architecture_migration_rules.sh` parses the
|
||||
allowed PR types from [`crate-boundaries.md`](crate-boundaries.md) and fails
|
||||
|
||||
@@ -126,6 +126,7 @@ tower-http = { workspace = true, features = ["trace", "compression-full", "cors"
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
flatbuffers.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
rustfs-signer.workspace = true
|
||||
|
||||
@@ -79,6 +79,7 @@ use crate::app::storage_compat::{
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
};
|
||||
use crate::server::convert_ecstore_object_info;
|
||||
use rustfs_concurrency::GetObjectQueueSnapshot;
|
||||
use rustfs_filemeta::{
|
||||
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateTargetDecision, ReplicationState, ReplicationStatusType,
|
||||
@@ -3663,11 +3664,11 @@ impl DefaultObjectUsecase {
|
||||
let event_args = EventArgsBuilder::new(
|
||||
event_name,
|
||||
notify_bucket.clone(),
|
||||
ObjectInfo {
|
||||
convert_ecstore_object_info(ObjectInfo {
|
||||
name: dobj.object_name.clone(),
|
||||
bucket: notify_bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
}),
|
||||
)
|
||||
.version_id(dobj.version_id.map(|v| v.to_string()).unwrap_or_default())
|
||||
.req_params(extract_params_header(&req_headers))
|
||||
@@ -4826,7 +4827,7 @@ impl DefaultObjectUsecase {
|
||||
let event_args = rustfs_notify::EventArgs {
|
||||
event_name: put_event_name_for_post_object(false),
|
||||
bucket_name: bucket.clone(),
|
||||
object: obj_info.clone().into(),
|
||||
object: convert_ecstore_object_info(obj_info.clone()),
|
||||
req_params: req_params.clone(),
|
||||
resp_elements: extract_resp_elements(&S3Response::new(output.clone())),
|
||||
version_id: version_id.clone(),
|
||||
|
||||
@@ -14,8 +14,10 @@
|
||||
|
||||
use super::{module_switch::resolve_notify_module_state, refresh_persisted_module_switches_from_store};
|
||||
use crate::app::context::resolve_server_config;
|
||||
use crate::storage::StorageObjectInfo;
|
||||
use crate::storage_compat::{EcstoreEventArgs, register_event_dispatch_hook};
|
||||
use rustfs_notify::EventArgs as NotifyEventArgs;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_notify::{EventArgs as NotifyEventArgs, NotifyObjectInfo};
|
||||
use rustfs_s3_types::EventName;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -38,6 +40,30 @@ pub fn is_notify_module_enabled() -> bool {
|
||||
NOTIFY_MODULE_ENABLED.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn convert_ecstore_object_info(object: StorageObjectInfo) -> NotifyObjectInfo {
|
||||
NotifyObjectInfo {
|
||||
bucket: object.bucket,
|
||||
name: object.name,
|
||||
size: object.size,
|
||||
etag: object.etag,
|
||||
content_type: object.content_type,
|
||||
user_defined: object
|
||||
.user_defined
|
||||
.iter()
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect(),
|
||||
version_id: object.version_id.map(|version_id| version_id.to_string()),
|
||||
mod_time: object
|
||||
.mod_time
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value.unix_timestamp(), value.nanosecond())),
|
||||
restore_expires: object
|
||||
.restore_expires
|
||||
.and_then(|value| DateTime::<Utc>::from_timestamp(value.unix_timestamp(), value.nanosecond())),
|
||||
storage_class: object.storage_class,
|
||||
transitioned_tier: (!object.transitioned_object.tier.is_empty()).then_some(object.transitioned_object.tier),
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option<NotifyEventArgs> {
|
||||
let version_id = args.object.version_id.map(|v| v.to_string()).unwrap_or_default();
|
||||
let (host, port) = parse_host_and_port(args.host);
|
||||
@@ -59,7 +85,7 @@ fn convert_ecstore_event_args(args: EcstoreEventArgs) -> Option<NotifyEventArgs>
|
||||
Some(NotifyEventArgs {
|
||||
event_name,
|
||||
bucket_name: args.bucket_name,
|
||||
object: args.object.into(),
|
||||
object: convert_ecstore_object_info(args.object),
|
||||
req_params,
|
||||
resp_elements,
|
||||
version_id,
|
||||
@@ -209,7 +235,12 @@ pub async fn init_event_notifier() {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::parse_host_and_port;
|
||||
use super::{convert_ecstore_object_info, parse_host_and_port};
|
||||
use crate::storage::StorageObjectInfo;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rustfs_storage_api::TransitionedObject;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
|
||||
#[test]
|
||||
fn parse_host_and_port_with_ipv4_and_port() {
|
||||
@@ -238,4 +269,40 @@ mod tests {
|
||||
assert_eq!(host, "localhost");
|
||||
assert_eq!(port, 9001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convert_ecstore_object_info_preserves_notify_event_fields() {
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + Duration::seconds(42);
|
||||
let restore_expires = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1_700_000_000);
|
||||
let mut metadata = HashMap::new();
|
||||
metadata.insert("x-amz-meta-key".to_string(), "value".to_string());
|
||||
|
||||
let converted = convert_ecstore_object_info(StorageObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 123,
|
||||
etag: Some("etag".to_string()),
|
||||
content_type: Some("text/plain".to_string()),
|
||||
user_defined: Arc::new(metadata),
|
||||
mod_time: Some(mod_time),
|
||||
restore_expires: Some(restore_expires),
|
||||
storage_class: Some("GLACIER".to_string()),
|
||||
transitioned_object: TransitionedObject {
|
||||
tier: "DEEP_ARCHIVE".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert_eq!(converted.bucket, "bucket");
|
||||
assert_eq!(converted.name, "object");
|
||||
assert_eq!(converted.size, 123);
|
||||
assert_eq!(converted.etag.as_deref(), Some("etag"));
|
||||
assert_eq!(converted.content_type.as_deref(), Some("text/plain"));
|
||||
assert_eq!(converted.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
|
||||
assert_eq!(converted.mod_time, DateTime::<Utc>::from_timestamp(42, 0));
|
||||
assert_eq!(converted.restore_expires, DateTime::<Utc>::from_timestamp(1_700_000_000, 0));
|
||||
assert_eq!(converted.storage_class.as_deref(), Some("GLACIER"));
|
||||
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ pub use service_state::ShutdownSignal;
|
||||
pub use service_state::wait_for_shutdown;
|
||||
|
||||
// Items only used within the library crate (admin handlers, server/http.rs, etc.).
|
||||
pub(crate) use event::convert_ecstore_object_info;
|
||||
pub(crate) use http::HeaderMapCarrier;
|
||||
pub(crate) use http::active_http_requests;
|
||||
pub(crate) use layer::RequestContextLayer;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::server::{is_audit_module_enabled, is_notify_module_enabled};
|
||||
use crate::server::{convert_ecstore_object_info, is_audit_module_enabled, is_notify_module_enabled};
|
||||
use crate::storage::access::{ReqInfo, request_context_from_req};
|
||||
use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
|
||||
use hashbrown::HashMap;
|
||||
@@ -206,7 +206,7 @@ impl OperationHelper {
|
||||
// initialize event builder
|
||||
// object is a placeholder that must be set later using the `object()` method.
|
||||
let event_builder = if notify_enabled {
|
||||
let mut event_builder = EventArgsBuilder::new(event, bucket, event_object)
|
||||
let mut event_builder = EventArgsBuilder::new(event, bucket, convert_ecstore_object_info(event_object))
|
||||
.host(get_request_host(&req.headers))
|
||||
.port(get_request_port(&req.headers))
|
||||
.user_agent(get_request_user_agent(&req.headers))
|
||||
@@ -241,7 +241,7 @@ impl OperationHelper {
|
||||
if let Self::Enabled(state) = &mut self
|
||||
&& let Some(builder) = state.event_builder.take()
|
||||
{
|
||||
state.event_builder = Some(builder.object(object_info));
|
||||
state.event_builder = Some(builder.object(convert_ecstore_object_info(object_info)));
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -533,11 +533,6 @@ cat >"$ECSTORE_OBJECT_API_EXTERNAL_ALIAS_EXPECTED_FILE" <<'EOF'
|
||||
crates/heal/src/heal/storage_compat.rs:HealObjectInfo=ObjectInfo
|
||||
crates/heal/src/heal/storage_compat.rs:HealObjectOptions=ObjectOptions
|
||||
crates/heal/src/heal/storage_compat.rs:HealPutObjReader=PutObjReader
|
||||
crates/iam/src/storage_compat.rs:IamConfigObjectInfo=ObjectInfo
|
||||
crates/iam/src/storage_compat.rs:IamConfigObjectOptions=ObjectOptions
|
||||
crates/iam/src/store/storage_compat.rs:IamObjectInfo=ObjectInfo
|
||||
crates/iam/src/store/storage_compat.rs:IamObjectOptions=ObjectOptions
|
||||
crates/notify/src/storage_compat.rs:EcstoreObjectInfo=ObjectInfo
|
||||
crates/protocols/src/swift/storage_compat.rs:SwiftGetObjectReader=GetObjectReader
|
||||
crates/protocols/src/swift/storage_compat.rs:SwiftObjectInfo=ObjectInfo
|
||||
crates/protocols/src/swift/storage_compat.rs:SwiftObjectOptions=ObjectOptions
|
||||
|
||||
Reference in New Issue
Block a user