mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-18 18:46:17 +00:00
refactor(replication): isolate filemeta facade contracts (#4243)
This commit is contained in:
@@ -35,6 +35,7 @@ paths.
|
|||||||
| `ReplicationStorage` | Object read/write/delete, object walk, metadata update, and target object IO. | ECStore object API, storage-api contracts, and read option types are concentrated in `replication_storage_boundary.rs`. |
|
| `ReplicationStorage` | Object read/write/delete, object walk, metadata update, and target object IO. | ECStore object API, storage-api contracts, and read option types are concentrated in `replication_storage_boundary.rs`. |
|
||||||
| `ReplicationMetadataStore` | Replication config, MRF/resync state, target reset headers, and status persistence. | Metadata sys access and replication metadata path constants are exposed through the contract type in `replication_metadata_boundary.rs`; versioning sys and config storage imports remain separate contracts. |
|
| `ReplicationMetadataStore` | Replication config, MRF/resync state, target reset headers, and status persistence. | Metadata sys access and replication metadata path constants are exposed through the contract type in `replication_metadata_boundary.rs`; versioning sys and config storage imports remain separate contracts. |
|
||||||
| `ReplicationResyncContracts` | Resync options, target status, bucket status, status classifiers, and persisted resync/MRF status wire format. | Owned by `crates/replication`; ECStore imports them through `replication_resync_boundary.rs`, which maps crate errors to ECStore errors. |
|
| `ReplicationResyncContracts` | Resync options, target status, bucket status, status classifiers, and persisted resync/MRF status wire format. | Owned by `crates/replication`; ECStore imports them through `replication_resync_boundary.rs`, which maps crate errors to ECStore errors. |
|
||||||
|
| `ReplicationCrateFileMetaFacade` | Replication facade compatibility symbols that still originate in filemeta wire contracts. | `crates/replication/src/filemeta.rs` is the only direct `rustfs-filemeta` import boundary inside `rustfs-replication`. |
|
||||||
| `ReplicationConfigStore` | Replication config persistence and config-derived labels used by target options. | Config read/save helpers and storage class labels are exposed through the contract type in `replication_config_store.rs`. |
|
| `ReplicationConfigStore` | Replication config persistence and config-derived labels used by target options. | Config read/save helpers and storage class labels are exposed through the contract type in `replication_config_store.rs`. |
|
||||||
| `ReplicationFileMeta` | Replication status, decisions, MRF entries, resync decisions, and target reset helpers. | `rustfs_filemeta` replication contracts are concentrated in `replication_filemeta_boundary.rs`; `FileInfo` remains in the storage boundary for storage trait bindings and walk options. |
|
| `ReplicationFileMeta` | Replication status, decisions, MRF entries, resync decisions, and target reset helpers. | `rustfs_filemeta` replication contracts are concentrated in `replication_filemeta_boundary.rs`; `FileInfo` remains in the storage boundary for storage trait bindings and walk options. |
|
||||||
| `StorageApiReplicationContracts` | Storage-api delete DTO replication state/status helpers. | The temporary `rustfs-filemeta` dependency is isolated in `crates/storage-api/src/replication.rs` until the wire contracts can move without creating a `rustfs-replication` / `rustfs-storage-api` cycle. |
|
| `StorageApiReplicationContracts` | Storage-api delete DTO replication state/status helpers. | The temporary `rustfs-filemeta` dependency is isolated in `crates/storage-api/src/replication.rs` until the wire contracts can move without creating a `rustfs-replication` / `rustfs-storage-api` cycle. |
|
||||||
@@ -87,6 +88,8 @@ paths.
|
|||||||
`crates/storage-api/src/replication.rs` until the underlying wire contracts
|
`crates/storage-api/src/replication.rs` until the underlying wire contracts
|
||||||
can move without a `rustfs-replication` / `rustfs-storage-api` dependency
|
can move without a `rustfs-replication` / `rustfs-storage-api` dependency
|
||||||
cycle.
|
cycle.
|
||||||
|
12. Keep direct `rustfs-filemeta` imports inside `rustfs-replication`
|
||||||
|
concentrated in `crates/replication/src/filemeta.rs`.
|
||||||
|
|
||||||
## First Code-Bearing Step
|
## First Code-Bearing Step
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
// Copyright 2024 RustFS Team
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
pub use rustfs_filemeta::{
|
||||||
|
MrfOpKind, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
|
||||||
|
REPLICATE_INCOMING_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos,
|
||||||
|
ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||||
|
ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType, get_replication_state,
|
||||||
|
parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||||
|
};
|
||||||
@@ -14,6 +14,7 @@
|
|||||||
|
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod delete;
|
pub mod delete;
|
||||||
|
mod filemeta;
|
||||||
pub mod mrf;
|
pub mod mrf;
|
||||||
pub mod multipart;
|
pub mod multipart;
|
||||||
pub mod object;
|
pub mod object;
|
||||||
@@ -33,6 +34,13 @@ pub use delete::{
|
|||||||
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
DeletedObjectReplicationInfo, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||||
should_retry_delete_marker_purge,
|
should_retry_delete_marker_purge,
|
||||||
};
|
};
|
||||||
|
pub use filemeta::{
|
||||||
|
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING_DELETE,
|
||||||
|
ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||||
|
ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision,
|
||||||
|
VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header,
|
||||||
|
version_purge_statuses_map,
|
||||||
|
};
|
||||||
pub use mrf::{MrfOpKind, MrfReplicateEntry, decode_mrf_file, encode_mrf_file};
|
pub use mrf::{MrfOpKind, MrfReplicateEntry, decode_mrf_file, encode_mrf_file};
|
||||||
pub use multipart::{
|
pub use multipart::{
|
||||||
ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
|
ReplicationMultipartPartInput, ReplicationMultipartPartPlan, ReplicationMultipartPlanError, ReplicationMultipartRange,
|
||||||
@@ -68,13 +76,6 @@ pub use runtime::{
|
|||||||
next_mrf_worker_count, next_regular_worker_count, replication_backpressure_recommendation, resized_worker_counts,
|
next_mrf_worker_count, next_regular_worker_count, replication_backpressure_recommendation, resized_worker_counts,
|
||||||
should_grow_large_workers, should_queue_large_object, worker_counts_for_priority,
|
should_grow_large_workers, should_queue_large_object, worker_counts_for_priority,
|
||||||
};
|
};
|
||||||
pub use rustfs_filemeta::{
|
|
||||||
REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING_DELETE,
|
|
||||||
ReplicateDecision, ReplicateObjectInfo, ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
|
||||||
ReplicationState, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision,
|
|
||||||
VersionPurgeStatusType, get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header,
|
|
||||||
version_purge_statuses_map,
|
|
||||||
};
|
|
||||||
pub use stats::{
|
pub use stats::{
|
||||||
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, BucketStats, ExponentialMovingAverage, FailStats,
|
ActiveWorkerStat, BucketReplicationStat, BucketReplicationStats, BucketStats, ExponentialMovingAverage, FailStats,
|
||||||
FailedMetric, InQueueMetric, InQueueStats, LatencyStats, ProxyMetric, ProxyStatsCache, QueueCache, QueueNode, QueueStats,
|
FailedMetric, InQueueMetric, InQueueStats, LatencyStats, ProxyMetric, ProxyStatsCache, QueueCache, QueueNode, QueueStats,
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use byteorder::{ByteOrder, LittleEndian};
|
|||||||
|
|
||||||
use crate::{Error, Result};
|
use crate::{Error, Result};
|
||||||
|
|
||||||
pub use rustfs_filemeta::{MrfOpKind, MrfReplicateEntry};
|
pub use crate::filemeta::{MrfOpKind, MrfReplicateEntry};
|
||||||
|
|
||||||
pub const MRF_META_FORMAT: u16 = 1;
|
pub const MRF_META_FORMAT: u16 = 1;
|
||||||
pub const MRF_META_VERSION: u16 = 1;
|
pub const MRF_META_VERSION: u16 = 1;
|
||||||
|
|||||||
@@ -12,8 +12,8 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
use crate::filemeta::{ReplicationAction, ReplicationType};
|
||||||
use crate::tagging::ReplicationTagFilter;
|
use crate::tagging::ReplicationTagFilter;
|
||||||
use rustfs_filemeta::{ReplicationAction, ReplicationType};
|
|
||||||
use rustfs_utils::http::{
|
use rustfs_utils::http::{
|
||||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING,
|
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING,
|
||||||
AMZ_WEBSITE_REDIRECT_LOCATION, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, EXPIRES,
|
AMZ_WEBSITE_REDIRECT_LOCATION, CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, EXPIRES,
|
||||||
@@ -164,7 +164,7 @@ mod tests {
|
|||||||
ReplicationSourceObject, ReplicationTargetObject, content_matches_by_etag, replication_action_for_target,
|
ReplicationSourceObject, ReplicationTargetObject, content_matches_by_etag, replication_action_for_target,
|
||||||
replication_etags_match, target_is_newer_than_source_null_version,
|
replication_etags_match, target_is_newer_than_source_null_version,
|
||||||
};
|
};
|
||||||
use rustfs_filemeta::{ReplicationAction, ReplicationType};
|
use crate::filemeta::{ReplicationAction, ReplicationType};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use time::{Duration, OffsetDateTime};
|
use time::{Duration, OffsetDateTime};
|
||||||
|
|
||||||
|
|||||||
@@ -125,6 +125,8 @@ Current coupling:
|
|||||||
- resync options, bucket/target resync status DTOs, status display labels, and
|
- resync options, bucket/target resync status DTOs, status display labels, and
|
||||||
the persisted resync status wire format live in `crates/replication`, with
|
the persisted resync status wire format live in `crates/replication`, with
|
||||||
ECStore retaining only error mapping and MRF persistence locally;
|
ECStore retaining only error mapping and MRF persistence locally;
|
||||||
|
- `crates/replication/src/filemeta.rs` is the only direct filemeta wire-contract
|
||||||
|
import boundary inside `rustfs-replication`;
|
||||||
- storage-api delete replication status/state helpers use the local
|
- storage-api delete replication status/state helpers use the local
|
||||||
`crates/storage-api/src/replication.rs` contract boundary while the
|
`crates/storage-api/src/replication.rs` contract boundary while the
|
||||||
underlying wire types remain in `rustfs-filemeta`;
|
underlying wire types remain in `rustfs-filemeta`;
|
||||||
@@ -171,6 +173,9 @@ Required contracts before crate movement:
|
|||||||
`crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs`,
|
`crates/ecstore/src/bucket/replication/replication_filemeta_boundary.rs`,
|
||||||
while `FileInfo` remains in the storage boundary for storage trait bindings
|
while `FileInfo` remains in the storage boundary for storage trait bindings
|
||||||
and walk options.
|
and walk options.
|
||||||
|
- `ReplicationCrateFileMetaFacade`: replication facade compatibility symbols
|
||||||
|
that still originate in filemeta wire contracts are concentrated in
|
||||||
|
`crates/replication/src/filemeta.rs` inside `rustfs-replication`.
|
||||||
- `StorageApiReplicationContracts`: storage-api delete DTO replication
|
- `StorageApiReplicationContracts`: storage-api delete DTO replication
|
||||||
state/status helpers are concentrated in `crates/storage-api/src/replication.rs`
|
state/status helpers are concentrated in `crates/storage-api/src/replication.rs`
|
||||||
until the underlying wire contracts can move without a
|
until the underlying wire contracts can move without a
|
||||||
|
|||||||
@@ -84,6 +84,7 @@ require_source_contains "docs/architecture/obs-ecstore-dependency-inventory.md"
|
|||||||
require_source_contains "docs/architecture/obs-ecstore-dependency-inventory.md" "crates/obs/src/metrics/storage_api.rs" "observability ECStore storage_api boundary"
|
require_source_contains "docs/architecture/obs-ecstore-dependency-inventory.md" "crates/obs/src/metrics/storage_api.rs" "observability ECStore storage_api boundary"
|
||||||
require_source_contains "docs/architecture/overview.md" "ecstore-api-facade-inventory.md" "architecture overview ECStore facade inventory link"
|
require_source_contains "docs/architecture/overview.md" "ecstore-api-facade-inventory.md" "architecture overview ECStore facade inventory link"
|
||||||
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ecstore-api-facade-inventory.md" "ECStore split plan facade inventory link"
|
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ecstore-api-facade-inventory.md" "ECStore split plan facade inventory link"
|
||||||
|
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "ReplicationCrateFileMetaFacade" "ECStore split plan replication crate filemeta facade section"
|
||||||
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "StorageApiReplicationContracts" "ECStore split plan storage-api replication contract section"
|
require_source_contains "docs/architecture/ecstore-module-split-plan.md" "StorageApiReplicationContracts" "ECStore split plan storage-api replication contract section"
|
||||||
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## Facade Group Inventory" "ECStore facade inventory group section"
|
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## Facade Group Inventory" "ECStore facade inventory group section"
|
||||||
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## External Consumer Boundaries" "ECStore facade inventory consumer boundary section"
|
require_source_contains "docs/architecture/ecstore-api-facade-inventory.md" "## External Consumer Boundaries" "ECStore facade inventory consumer boundary section"
|
||||||
@@ -203,6 +204,7 @@ FUZZ_ECSTORE_COMPAT_BYPASS_HITS_FILE="${TMP_DIR}/fuzz_ecstore_compat_bypass_hits
|
|||||||
EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt"
|
EXTERNAL_ECSTORE_API_BOUNDARY_HITS_FILE="${TMP_DIR}/external_ecstore_api_boundary_hits.txt"
|
||||||
REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt"
|
REPLICATION_FACADE_BYPASS_HITS_FILE="${TMP_DIR}/replication_facade_bypass_hits.txt"
|
||||||
REPLICATION_FACADE_WILDCARD_EXPORT_HITS_FILE="${TMP_DIR}/replication_facade_wildcard_export_hits.txt"
|
REPLICATION_FACADE_WILDCARD_EXPORT_HITS_FILE="${TMP_DIR}/replication_facade_wildcard_export_hits.txt"
|
||||||
|
REPLICATION_CRATE_FILEMETA_BYPASS_HITS_FILE="${TMP_DIR}/replication_crate_filemeta_bypass_hits.txt"
|
||||||
REPLICATION_CONFIG_RULE_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_config_rule_contract_backslide_hits.txt"
|
REPLICATION_CONFIG_RULE_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_config_rule_contract_backslide_hits.txt"
|
||||||
REPLICATION_DELETE_WORKER_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_delete_worker_contract_backslide_hits.txt"
|
REPLICATION_DELETE_WORKER_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_delete_worker_contract_backslide_hits.txt"
|
||||||
REPLICATION_OPERATION_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_operation_contract_backslide_hits.txt"
|
REPLICATION_OPERATION_CONTRACT_BACKSLIDE_HITS_FILE="${TMP_DIR}/replication_operation_contract_backslide_hits.txt"
|
||||||
@@ -2607,6 +2609,18 @@ if [[ -s "$REPLICATION_FACADE_WILDCARD_EXPORT_HITS_FILE" ]]; then
|
|||||||
report_failure "replication facade must use explicit compatibility exports instead of wildcard re-exports: $(paste -sd '; ' "$REPLICATION_FACADE_WILDCARD_EXPORT_HITS_FILE")"
|
report_failure "replication facade must use explicit compatibility exports instead of wildcard re-exports: $(paste -sd '; ' "$REPLICATION_FACADE_WILDCARD_EXPORT_HITS_FILE")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --with-filename 'rustfs_filemeta::|use\s+rustfs_filemeta\b' \
|
||||||
|
crates/replication/src \
|
||||||
|
--glob '*.rs' |
|
||||||
|
rg -v '^crates/replication/src/filemeta\.rs:' || true
|
||||||
|
) >"$REPLICATION_CRATE_FILEMETA_BYPASS_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$REPLICATION_CRATE_FILEMETA_BYPASS_HITS_FILE" ]]; then
|
||||||
|
report_failure "replication crate filemeta contracts must stay behind crates/replication/src/filemeta.rs: $(paste -sd '; ' "$REPLICATION_CRATE_FILEMETA_BYPASS_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
replication_config_rule_status=0
|
replication_config_rule_status=0
|
||||||
|
|||||||
Reference in New Issue
Block a user