feat(connect): collect bounded offline diagnostics (#6450)

* feat(connect): collect bounded offline diagnostics

* fix(connect): tighten offline diagnostic boundaries

rustfs/tests/connect_offline_collectors.rs

@@ -0,0 +1,290 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Offline collector and frozen redaction conformance tests.
+
+use std::fs;
+use std::path::PathBuf;
+
+use rustfs::connect::offline::{CollectorError, RedactionSource, collect_offline_diagnostics, redact_json};
+use rustfs_madmin::{Disk, ITEM_OFFLINE, StorageInfo};
+use serde_json::{Map, Value, json};
+use sha2::{Digest as _, Sha256};
+use tokio_util::sync::CancellationToken;
+
+fn fixture_dir() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../protocol/agent/v1/fixtures/redaction")
+}
+
+fn fixture_json(name: &str) -> Value {
+    let manifest = fs::read_to_string(fixture_dir().join("MANIFEST.sha256")).expect("read fixture manifest");
+    let expected = manifest
+        .lines()
+        .find_map(|line| {
+            let (digest, file) = line.split_once("  ")?;
+            (file == name).then_some(digest)
+        })
+        .unwrap_or_else(|| panic!("{name} is listed in the fixture manifest"));
+    let bytes = fs::read(fixture_dir().join(name)).unwrap_or_else(|error| panic!("read {name}: {error}"));
+    let actual = Sha256::digest(&bytes)
+        .iter()
+        .map(|byte| format!("{byte:02x}"))
+        .collect::<String>();
+    assert_eq!(actual, expected, "{name} matches the frozen manifest");
+    serde_json::from_slice(&bytes).unwrap_or_else(|error| panic!("parse {name}: {error}"))
+}
+
+fn vectors(name: &str) -> Vec<Value> {
+    fixture_json(name)["vectors"].as_array().expect("fixture vectors").clone()
+}
+
+fn object(value: &Value) -> Map<String, Value> {
+    value.as_object().expect("fixture document is an object").clone()
+}
+
+fn redact_document(source: RedactionSource, document: &Map<String, Value>) -> rustfs::connect::offline::RedactionResult {
+    let encoded = serde_json::to_vec(document).expect("fixture document is representable");
+    redact_json(source, &encoded).expect("fixture document must be accepted")
+}
+
+#[test]
+fn connect_offline_collectors_match_every_allowed_and_secret_redaction_vector() {
+    let ruleset = fixture_json("ruleset.json");
+    for fixture in ["allowed-vectors.json", "secret-vectors.json"] {
+        for vector in vectors(fixture) {
+            let name = vector["name"].as_str().expect("vector name");
+            let source = RedactionSource::try_from(vector["source"].as_str().expect("vector source"))
+                .unwrap_or_else(|error| panic!("{name}: {error}"));
+            let result = redact_document(source, &object(&vector["document"]));
+            assert_eq!(result.redaction_version, ruleset["redactionVersion"], "{name}: redaction version");
+            assert_eq!(result.ruleset_hash, ruleset["rulesetHash"], "{name}: ruleset hash");
+            assert_eq!(result.canonical_json, vector["expectedCanonicalJson"], "{name}: canonical bytes");
+            if let Some(expected) = vector["expectedRedactedCount"].as_u64() {
+                assert_eq!(result.redacted_count as u64, expected, "{name}: redacted count");
+                assert_eq!(result.counts.dropped_field as u64, vector["expectedCounts"]["droppedField"], "{name}");
+                assert_eq!(result.counts.redacted_value as u64, vector["expectedCounts"]["redactedValue"], "{name}");
+                assert_eq!(
+                    result.counts.redacted_oversize_value as u64, vector["expectedCounts"]["redactedOversizeValue"],
+                    "{name}"
+                );
+            } else {
+                assert_eq!(result.redacted_count, 0, "{name}: allowed vectors must not be changed");
+            }
+            if let Some(secrets) = vector["secretLiterals"].as_array() {
+                for secret in secrets {
+                    assert!(
+                        !result.canonical_json.contains(secret.as_str().expect("secret literal")),
+                        "{name}: a secret survived redaction"
+                    );
+                }
+            }
+        }
+    }
+}
+
+#[test]
+fn connect_offline_collectors_enforce_the_frozen_rejection_budgets() {
+    for vector in vectors("rejection-vectors.json") {
+        let name = vector["name"].as_str().expect("vector name");
+        let expected = vector["expected"]["message"].as_str();
+        let source = match RedactionSource::try_from(vector["source"].as_str().expect("source")) {
+            Ok(source) => source,
+            Err(error) => {
+                assert_eq!(error.to_string(), expected.expect("refusal message"), "{name}");
+                continue;
+            }
+        };
+        let build = &vector["build"];
+        let field = build["field"].as_str().unwrap_or("rustfsVersion");
+        let document = match build["kind"].as_str().expect("builder kind") {
+            "literal" => object(&build["document"]),
+            "bulkStrings" => {
+                let entries = build["entries"].as_u64().expect("entries") as usize;
+                let bytes = build["valueBytes"].as_u64().expect("value bytes") as usize;
+                let nested = (0..entries)
+                    .map(|index| (format!("f{index}"), json!("a".repeat(bytes))))
+                    .collect();
+                Map::from_iter([(field.to_owned(), Value::Object(nested))])
+            }
+            "nestedDepth" => {
+                let mut nested = json!({ "leaf": 1 });
+                for _ in 0..build["depth"].as_u64().expect("depth") {
+                    nested = json!({ "nested": nested });
+                }
+                Map::from_iter([(field.to_owned(), nested)])
+            }
+            "listNodes" => {
+                let count = build["count"].as_u64().expect("count") as usize;
+                Map::from_iter([(field.to_owned(), Value::Array(vec![json!(1); count]))])
+            }
+            "unrepresentable" => {
+                let encoded = format!(r#"{{"{field}":NaN}}"#);
+                assert_eq!(
+                    redact_json(source, encoded.as_bytes()).expect_err(name).to_string(),
+                    expected.expect("refusal message"),
+                    "{name}"
+                );
+                continue;
+            }
+            kind => panic!("unknown fixture builder {kind}"),
+        };
+        match expected {
+            Some(message) => {
+                let encoded = serde_json::to_vec(&document).expect("rejection fixture document is representable");
+                assert_eq!(redact_json(source, &encoded).expect_err(name).to_string(), message, "{name}")
+            }
+            None => assert_eq!(
+                redact_document(source, &document).redacted_count,
+                vector["expected"]["redactedCount"],
+                "{name}"
+            ),
+        }
+    }
+}
+
+#[test]
+fn connect_offline_collectors_reject_oversize_raw_input_before_parsing() {
+    let invalid = vec![b'!'; 262_145];
+    assert_eq!(
+        redact_json(RedactionSource::OfflineDiagnostic, &invalid)
+            .expect_err("oversize raw input")
+            .to_string(),
+        "Redaction refused the document: its size in bytes exceeds the frozen budget of 262144."
+    );
+}
+
+#[tokio::test]
+async fn connect_offline_collectors_emit_only_fixed_redacted_entries_and_honor_cancellation() {
+    let storage = StorageInfo {
+        disks: vec![
+            Disk {
+                endpoint: "https://node-a.private.example:9000/data-a".to_owned(),
+                drive_path: "/secret/customer/path-a".to_owned(),
+                uuid: "private-drive-a".to_owned(),
+                state: "ok".to_owned(),
+                total_space: 1_000,
+                used_space: 400,
+                ..Disk::default()
+            },
+            Disk {
+                endpoint: "https://node-a.private.example:9000/data-b".to_owned(),
+                drive_path: "/secret/customer/path-b".to_owned(),
+                uuid: "private-drive-b".to_owned(),
+                state: "unformatted".to_owned(),
+                total_space: 2_000,
+                used_space: 500,
+                ..Disk::default()
+            },
+            Disk {
+                endpoint: "https://node-b.private.example:9000/data-c".to_owned(),
+                drive_path: "/secret/customer/path-c".to_owned(),
+                uuid: "private-drive-c".to_owned(),
+                state: ITEM_OFFLINE.to_owned(),
+                total_space: 3_000,
+                used_space: 600,
+                healing: true,
+                ..Disk::default()
+            },
+        ],
+        ..StorageInfo::default()
+    };
+    let cancel = CancellationToken::new();
+    let entries = collect_offline_diagnostics(&storage, &cancel)
+        .await
+        .expect("collect fixed offline entries");
+    assert_eq!(entries.len(), 12);
+    let encoded = serde_json::to_string(&entries).expect("manifest entries serialize");
+    for forbidden in [
+        "node-a.private.example",
+        "node-b.private.example",
+        "/secret/customer/path-a",
+        "/secret/customer/path-b",
+        "/secret/customer/path-c",
+        "private-drive-a",
+        "private-drive-b",
+        "private-drive-c",
+    ] {
+        assert!(!encoded.contains(forbidden), "private storage metadata must not leave the collector");
+    }
+    assert!(entries.iter().all(|entry| entry.field_id.starts_with("offline.")));
+    assert!(entries.iter().all(|entry| entry.canonical_json.len() <= 16 * 1024));
+    let canonical = |field_id| {
+        entries
+            .iter()
+            .find(|entry| entry.field_id == field_id)
+            .unwrap_or_else(|| panic!("missing {field_id}"))
+            .canonical_json
+            .as_str()
+    };
+    assert_eq!(canonical("offline.nodeCount"), r#"{"nodeCount":2}"#);
+    assert_eq!(canonical("offline.driveCount"), r#"{"driveCount":3}"#);
+    assert_eq!(canonical("offline.capacityUsedBytes"), r#"{"capacityUsedBytes":1500}"#);
+    assert_eq!(canonical("offline.capacityTotalBytes"), r#"{"capacityTotalBytes":6000}"#);
+    assert_eq!(
+        canonical("offline.coarseHealthFlags"),
+        r#"{"coarseHealthFlags":{"degraded":true,"healing":true,"offlineDrives":1,"scanning":false}}"#
+    );
+
+    let healthy = StorageInfo {
+        disks: ["ok", "unformatted", "online"]
+            .into_iter()
+            .enumerate()
+            .map(|(index, state)| Disk {
+                endpoint: format!("https://healthy.example:9000/data-{index}"),
+                state: state.to_owned(),
+                ..Disk::default()
+            })
+            .collect(),
+        ..StorageInfo::default()
+    };
+    let healthy_entries = collect_offline_diagnostics(&healthy, &CancellationToken::new())
+        .await
+        .expect("collect healthy storage summary");
+    assert_eq!(
+        healthy_entries
+            .iter()
+            .find(|entry| entry.field_id == "offline.coarseHealthFlags")
+            .expect("healthy coarse health entry")
+            .canonical_json,
+        r#"{"coarseHealthFlags":{"degraded":false,"healing":false,"offlineDrives":0,"scanning":false}}"#
+    );
+
+    cancel.cancel();
+    assert!(matches!(
+        collect_offline_diagnostics(&storage, &cancel).await,
+        Err(CollectorError::Cancelled)
+    ));
+
+    let oversized = StorageInfo {
+        disks: vec![Disk::default(); 4_097],
+        ..StorageInfo::default()
+    };
+    let active = CancellationToken::new();
+    assert!(matches!(
+        collect_offline_diagnostics(&oversized, &active).await,
+        Err(CollectorError::StorageTopologyTooLarge)
+    ));
+
+    let invalid_endpoint = StorageInfo {
+        disks: vec![Disk {
+            endpoint: "not-an-endpoint".to_owned(),
+            ..Disk::default()
+        }],
+        ..StorageInfo::default()
+    };
+    assert!(matches!(
+        collect_offline_diagnostics(&invalid_endpoint, &active).await,
+        Err(CollectorError::InvalidStorageEndpoint)
+    ));
+}