mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-24 13:16:28 +00:00
refactor: prune storage compatibility re-export allowances (#3579)
* refactor: prune storage compatibility re-export allowances * fix: reject whitespace-padded dot path segments
This commit is contained in:
@@ -137,25 +137,9 @@ pub fn has_bad_path_component(path: &str) -> bool {
|
|||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Trim whitespace of segment
|
// Trim whitespace of segment and check for ".." or "."
|
||||||
let mut segment_start = start;
|
match path[start..i].trim() {
|
||||||
let mut segment_end = i;
|
"." | ".." => return true,
|
||||||
|
|
||||||
while segment_start < segment_end && bytes[segment_start].is_ascii_whitespace() {
|
|
||||||
segment_start += 1;
|
|
||||||
}
|
|
||||||
while segment_end > segment_start && bytes[segment_end - 1].is_ascii_whitespace() {
|
|
||||||
segment_end -= 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for ".." or "."
|
|
||||||
match segment_end - segment_start {
|
|
||||||
2 if segment_start + 1 < n && bytes[segment_start] == b'.' && bytes[segment_start + 1] == b'.' => {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
1 if bytes[segment_start] == b'.' => {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -436,6 +420,9 @@ mod tests {
|
|||||||
assert!(!is_valid_object_prefix("prefix/./other"));
|
assert!(!is_valid_object_prefix("prefix/./other"));
|
||||||
assert!(!is_valid_object_prefix("a/../b/../c"));
|
assert!(!is_valid_object_prefix("a/../b/../c"));
|
||||||
assert!(!is_valid_object_prefix("a/./b/./c"));
|
assert!(!is_valid_object_prefix("a/./b/./c"));
|
||||||
|
assert!(!is_valid_object_prefix("\x0b./object"));
|
||||||
|
assert!(!is_valid_object_prefix("prefix/\x0b../object"));
|
||||||
|
assert!(!is_valid_object_prefix("\x0b.\\\\object"));
|
||||||
|
|
||||||
// Invalid cases - double slashes
|
// Invalid cases - double slashes
|
||||||
assert!(!is_valid_object_prefix("prefix//with//double//slashes"));
|
assert!(!is_valid_object_prefix("prefix//with//double//slashes"));
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{data_usage, disk, error, global, store, store_api};
|
pub(crate) use rustfs_ecstore::{data_usage, disk, error, global, store, store_api};
|
||||||
}
|
}
|
||||||
pub(crate) use self::ecstore::{
|
pub(crate) use self::ecstore::{
|
||||||
|
|||||||
@@ -13,7 +13,5 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{config, error, global, notification_sys, store};
|
pub(crate) use rustfs_ecstore::{config, error, global, notification_sys, store};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{config, global, store_api};
|
pub(crate) use rustfs_ecstore::{config, global, store_api};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,5 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{bucket, data_usage, global, pools, resolve_object_store_handle};
|
pub(crate) use rustfs_ecstore::{bucket, data_usage, global, pools, resolve_object_store_handle};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
mod ecstore {
|
mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(super) use rustfs_ecstore::{bucket, error, resolve_object_store_handle, store, store_api};
|
pub(super) use rustfs_ecstore::{bucket, error, resolve_object_store_handle, store, store_api};
|
||||||
}
|
}
|
||||||
use self::ecstore::{
|
use self::ecstore::{
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{error, resolve_object_store_handle, set_disk, store, store_api};
|
pub(crate) use rustfs_ecstore::{error, resolve_object_store_handle, set_disk, store, store_api};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,8 +14,6 @@
|
|||||||
|
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{
|
pub(crate) use rustfs_ecstore::{
|
||||||
bucket, cache_value, config, data_usage, disk, error, global, pools, resolve_object_store_handle, set_disk, store,
|
bucket, cache_value, config, data_usage, disk, error, global, pools, resolve_object_store_handle, set_disk, store,
|
||||||
store_api, store_utils,
|
store_api, store_utils,
|
||||||
|
|||||||
@@ -13,7 +13,5 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{bucket, client, disk, endpoints, global, pools, store, tier};
|
pub(crate) use rustfs_ecstore::{bucket, client, disk, endpoints, global, pools, store, tier};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,18 +5,18 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
## Current Context
|
## Current Context
|
||||||
|
|
||||||
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
|
||||||
- Branch: `overtrue/arch-remaining-storage-compat-exports`
|
- Branch: `overtrue/arch-compat-reexport-prune`
|
||||||
- Baseline: `rustfs/rustfs#3576` head
|
- Baseline: `origin/main` after `rustfs/rustfs#3578`
|
||||||
`8255bd68539dc3c8cfc07a0ceb783ead67a02453`.
|
(`f3fcdd4ba26181e3fe5d3afb6d6f5510bb9ad186`).
|
||||||
- PR type for this branch: `consumer-migration`
|
- PR type for this branch: `consumer-migration`
|
||||||
- Runtime behavior changes: no migration behavior change expected.
|
- Runtime behavior changes: no migration behavior change expected.
|
||||||
- Rust code changes: narrow remaining scanner, heal, Swift, and IAM store
|
- Rust code changes: prune unused ECStore compatibility re-export allowances
|
||||||
ECStore compatibility boundary modules from direct ECStore imports to explicit
|
from production and fuzz compatibility boundaries, keep target-specific test
|
||||||
local `ecstore` re-export surfaces. ECStore-owned definitions and runtime
|
harness exceptions explicit, and gate test-only RustFS storage compatibility
|
||||||
behavior stay in ECStore.
|
re-exports with `cfg(test)`.
|
||||||
- CI/script changes: none.
|
- CI/script changes: add a migration guard rejecting production
|
||||||
- Docs changes: record the remaining compatibility export surface cleanup
|
`storage_compat.rs` unused-import allows.
|
||||||
slice.
|
- Docs changes: record the compatibility re-export pruning slice.
|
||||||
|
|
||||||
## Phase 0 Tasks
|
## Phase 0 Tasks
|
||||||
|
|
||||||
@@ -40,7 +40,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
public storage-api re-export coverage, ECStore compatibility-test coverage,
|
public storage-api re-export coverage, ECStore compatibility-test coverage,
|
||||||
and a production-source guard against reintroducing the removed
|
and a production-source guard against reintroducing the removed
|
||||||
`StorageAPI` aggregate facade identifier; add a source guard that rejects
|
`StorageAPI` aggregate facade identifier; add a source guard that rejects
|
||||||
direct `rustfs_ecstore` imports outside compatibility boundary modules.
|
direct `rustfs_ecstore` imports outside compatibility boundary modules; add
|
||||||
|
a guard that rejects production compatibility boundaries hiding unused
|
||||||
|
ECStore re-exports.
|
||||||
- Acceptance: architecture migration rules fail if the public storage-api
|
- Acceptance: architecture migration rules fail if the public storage-api
|
||||||
contract re-export surface drifts or if ECStore compile-time compatibility
|
contract re-export surface drifts or if ECStore compile-time compatibility
|
||||||
tests for the remaining storage-admin and namespace-lock contracts are
|
tests for the remaining storage-admin and namespace-lock contracts are
|
||||||
@@ -963,6 +965,28 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
migration/layer guards, formatting check, diff hygiene, direct import scan,
|
migration/layer guards, formatting check, diff hygiene, direct import scan,
|
||||||
risk scan, full pre-commit, and required three-expert review passed.
|
risk scan, full pre-commit, and required three-expert review passed.
|
||||||
|
|
||||||
|
- [x] `API-035` Prune compatibility re-export allowances.
|
||||||
|
- Current branch: `overtrue/arch-compat-reexport-prune`.
|
||||||
|
- Current slice: remove unused-import allowances from production and fuzz
|
||||||
|
ECStore compatibility boundary modules, keep target-specific test harness
|
||||||
|
exceptions explicit, gate test-only RustFS storage compatibility re-exports
|
||||||
|
with `cfg(test)`, and add a migration rule preventing production
|
||||||
|
compatibility boundaries from hiding unused ECStore re-exports.
|
||||||
|
- Acceptance: production and fuzz `storage_compat.rs` modules compile without
|
||||||
|
unused-import allows, test-only compatibility exceptions remain scoped to
|
||||||
|
harnesses with target-specific compile needs, and migration rules reject
|
||||||
|
reintroducing broad unused-import allowances in production compatibility
|
||||||
|
boundaries.
|
||||||
|
- Must preserve: all ECStore-owned concrete types and runtime behavior,
|
||||||
|
startup/storage/admin/app/Swift/scanner/heal/IAM/notify/obs/S3 Select
|
||||||
|
import paths, test harness behavior, and fuzz target behavior.
|
||||||
|
- Risk defense: this slice changes only compatibility boundary re-export
|
||||||
|
hygiene and migration guard coverage; it does not move definitions, alter
|
||||||
|
runtime control flow, mutate metadata formats, or change storage behavior.
|
||||||
|
- Verification: focused compile checks, fuzz manifest compile, migration and
|
||||||
|
layer guards, formatting check, diff hygiene, risk scan, full pre-commit,
|
||||||
|
and required three-expert review passed.
|
||||||
|
|
||||||
## Phase 8 Background Controller Tasks
|
## Phase 8 Background Controller Tasks
|
||||||
|
|
||||||
- [x] `BGC-001` Inventory background services.
|
- [x] `BGC-001` Inventory background services.
|
||||||
@@ -1239,9 +1263,9 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
|
|||||||
|
|
||||||
| Expert | Status | Notes |
|
| Expert | Status | Notes |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| Quality/architecture | passed | Scanner, heal, Swift, and IAM store now keep direct ECStore references inside local compatibility boundary modules with existing local semantic aliases preserved; guard coverage rejects new bypasses. |
|
| Quality/architecture | passed | Production compatibility boundaries no longer hide unused ECStore re-exports behind broad unused-import allows; target-specific test exceptions remain scoped. |
|
||||||
| Migration preservation | passed | ECStore remains the owner of backing types and behavior; scanner, heal, Swift, and IAM runtime semantics are unchanged. |
|
| Migration preservation | passed | ECStore remains the owner of backing types and behavior; this slice only changes compatibility-boundary hygiene and guard coverage. |
|
||||||
| Testing/verification | passed | Focused scanner/heal/IAM compile, Swift feature compile, guards, formatting check, diff hygiene, risk scan, and full `make pre-commit` passed. |
|
| Testing/verification | passed | Focused compile checks, fuzz manifest compile, guards, formatting, diff hygiene, risk scan, and full `make pre-commit` passed. |
|
||||||
|
|
||||||
## Verification Notes
|
## Verification Notes
|
||||||
|
|
||||||
@@ -1250,6 +1274,9 @@ Passed before push:
|
|||||||
- `cargo check --tests -p rustfs-scanner -p rustfs-heal -p rustfs-iam`:
|
- `cargo check --tests -p rustfs-scanner -p rustfs-heal -p rustfs-iam`:
|
||||||
passed.
|
passed.
|
||||||
- `cargo check --tests -p rustfs-protocols --features swift`: passed.
|
- `cargo check --tests -p rustfs-protocols --features swift`: passed.
|
||||||
|
- `cargo check --tests -p rustfs -p rustfs-scanner -p rustfs-heal -p rustfs-iam -p rustfs-notify -p rustfs-obs -p rustfs-s3select-api -p e2e_test`:
|
||||||
|
passed.
|
||||||
|
- `cargo check --manifest-path fuzz/Cargo.toml --bins`: passed.
|
||||||
- `rg -n 'rustfs_ecstore' crates/scanner/src crates/heal/src crates/protocols/src/swift crates/iam/src/store --glob '*.rs'`:
|
- `rg -n 'rustfs_ecstore' crates/scanner/src crates/heal/src crates/protocols/src/swift crates/iam/src/store --glob '*.rs'`:
|
||||||
remaining matches are deliberate compatibility boundary definitions.
|
remaining matches are deliberate compatibility boundary definitions.
|
||||||
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
- `./scripts/check_architecture_migration_rules.sh`: passed.
|
||||||
|
|||||||
@@ -13,7 +13,5 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::bucket;
|
pub(crate) use rustfs_ecstore::bucket;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{
|
pub(crate) use rustfs_ecstore::{
|
||||||
bucket, client, config, data_usage, disk, endpoints, error, global, metrics_realtime, notification_sys, rebalance, rpc,
|
bucket, client, config, data_usage, disk, endpoints, error, global, metrics_realtime, notification_sys, rebalance, rpc,
|
||||||
store, store_utils, tier,
|
store, store_utils, tier,
|
||||||
|
|||||||
@@ -13,8 +13,6 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{
|
pub(crate) use rustfs_ecstore::{
|
||||||
admin_server_info, bucket, client, compress, config, data_usage, disk, endpoints, error, global, new_object_layer_fn,
|
admin_server_info, bucket, client, compress, config, data_usage, disk, endpoints, error, global, new_object_layer_fn,
|
||||||
notification_sys, pools, rio, set_disk, set_object_store_resolver, store, tier,
|
notification_sys, pools, rio, set_disk, set_object_store_resolver, store, tier,
|
||||||
|
|||||||
@@ -13,10 +13,11 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{
|
pub(crate) use rustfs_ecstore::{
|
||||||
admin_server_info, bucket, client, config, disk, error, get_global_lock_client, global, metrics_realtime,
|
admin_server_info, bucket, client, disk, error, get_global_lock_client, global, metrics_realtime,
|
||||||
resolve_object_store_handle, rio, rpc, set_disk, store, store_api,
|
resolve_object_store_handle, rio, rpc, set_disk, store, store_api,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::config;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,10 +13,11 @@
|
|||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
pub(crate) mod ecstore {
|
pub(crate) mod ecstore {
|
||||||
#![allow(unused_imports)]
|
|
||||||
|
|
||||||
pub(crate) use rustfs_ecstore::{
|
pub(crate) use rustfs_ecstore::{
|
||||||
bucket, config, data_usage, disk, disks_layout, endpoints, error, event_notification, global, notification_sys, pools,
|
bucket, config, disk, endpoints, error, event_notification, global, notification_sys, resolve_object_store_handle, rpc,
|
||||||
resolve_object_store_handle, rpc, set_disk, set_global_endpoints, store, update_erasure_type,
|
set_disk, set_global_endpoints, store, update_erasure_type,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) use rustfs_ecstore::disks_layout;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ STORE_API_EXTERNAL_LIST_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_list_c
|
|||||||
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
STORE_API_EXTERNAL_OPERATION_CONSUMER_HITS_FILE="${TMP_DIR}/store_api_external_operation_consumer_hits.txt"
|
||||||
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
STORE_API_OBJECT_OPERATION_LOCAL_METHOD_HITS_FILE="${TMP_DIR}/store_api_object_operation_local_method_hits.txt"
|
||||||
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
|
DIRECT_ECSTORE_IMPORT_HITS_FILE="${TMP_DIR}/direct_ecstore_import_hits.txt"
|
||||||
|
PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE="${TMP_DIR}/production_unused_compat_allow_hits.txt"
|
||||||
|
|
||||||
awk '
|
awk '
|
||||||
/^## PR Types$/ {
|
/^## PR Types$/ {
|
||||||
@@ -334,6 +335,19 @@ if [[ -s "$DIRECT_ECSTORE_IMPORT_HITS_FILE" ]]; then
|
|||||||
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
|
report_failure "direct rustfs_ecstore imports outside compatibility boundaries are forbidden: $(paste -sd '; ' "$DIRECT_ECSTORE_IMPORT_HITS_FILE")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
(
|
||||||
|
cd "$ROOT_DIR"
|
||||||
|
rg -n --no-heading '#!\[allow\(unused_imports\)\]' \
|
||||||
|
rustfs/src crates/*/src fuzz/fuzz_targets \
|
||||||
|
--glob '*storage_compat.rs' \
|
||||||
|
--glob '!crates/ecstore/**' \
|
||||||
|
--glob '!crates/e2e_test/**' || true
|
||||||
|
) >"$PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE"
|
||||||
|
|
||||||
|
if [[ -s "$PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE" ]]; then
|
||||||
|
report_failure "production storage compatibility boundaries must not hide unused ECStore re-exports: $(paste -sd '; ' "$PRODUCTION_UNUSED_COMPAT_ALLOW_HITS_FILE")"
|
||||||
|
fi
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR"
|
cd "$ROOT_DIR"
|
||||||
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
rg -n --no-heading 'async fn (get_object_reader|put_object|get_object_info|verify_object_integrity|copy_object|delete_object_version|delete_object|delete_objects|put_object_metadata|get_object_tags|put_object_tags|delete_object_tags|add_partial|transition_object|restore_transitioned_object|list_multipart_uploads|new_multipart_upload|copy_object_part|put_object_part|get_multipart_info|list_object_parts|abort_multipart_upload|complete_multipart_upload|heal_format|heal_bucket|heal_object|get_pool_and_set|check_abandoned_parts|new_ns_lock)\b' \
|
||||||
|
|||||||
Reference in New Issue
Block a user