mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-21 20:06:37 +00:00
fix(kms): make manifest digest canonicalization independent of map ordering
The canonical digest form serialized serde_json values directly, which inherits the key order of serde_json's map type: sorted by default, but insertion-ordered when any crate in the unified build graph enables the preserve_order feature. The workspace-wide CI build unified that feature while a per-crate local build did not, so the frozen fixture digest matched in one environment and not the other — and a bundle sealed by one build flavor would fail verification in the other. Canonicalization now rebuilds every JSON object with bytewise-sorted keys (array order preserved) before hashing, so the digest bytes are identical regardless of feature unification. Reproduced by enabling preserve_order in dev-dependencies (fixture test red), then verified green with the fix under both map flavors.
This commit is contained in:
@@ -462,16 +462,19 @@ impl BackupManifest {
|
|||||||
/// Compute the digest over the canonical manifest form.
|
/// Compute the digest over the canonical manifest form.
|
||||||
///
|
///
|
||||||
/// Canonical form: the JSON *value* of the manifest with the digest hex
|
/// Canonical form: the JSON *value* of the manifest with the digest hex
|
||||||
/// emptied, serialized compactly by `serde_json`'s value serializer. The
|
/// emptied, object keys rebuilt in bytewise-sorted order at every level
|
||||||
/// value layer is what makes sealing and decoding agree byte-for-byte:
|
/// (see [`canonicalize_value`]), then serialized compactly. The value
|
||||||
/// a decoder recovers the identical value from the raw stored bytes
|
/// layer is what makes sealing and decoding agree byte-for-byte — a
|
||||||
/// without round-tripping any typed field through parse-and-reprint.
|
/// decoder recovers the identical value from the raw stored bytes
|
||||||
|
/// without round-tripping any typed field through parse-and-reprint —
|
||||||
|
/// and the explicit key sort makes the bytes independent of
|
||||||
|
/// `serde_json`'s map implementation (`preserve_order` on or off).
|
||||||
pub fn compute_digest(&self) -> Result<ContentDigest, BackupError> {
|
pub fn compute_digest(&self) -> Result<ContentDigest, BackupError> {
|
||||||
let mut unsealed = self.clone();
|
let mut unsealed = self.clone();
|
||||||
unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm);
|
unsealed.manifest_digest = ContentDigest::placeholder(self.manifest_digest.algorithm);
|
||||||
let value = serde_json::to_value(&unsealed)
|
let value = serde_json::to_value(&unsealed)
|
||||||
.map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
|
.map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
|
||||||
Self::digest_of_canonical_value(&value, self.manifest_digest.algorithm)
|
Self::digest_of_canonical_value(value, self.manifest_digest.algorithm)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verify the sealed digest against the current in-memory content.
|
/// Verify the sealed digest against the current in-memory content.
|
||||||
@@ -503,7 +506,7 @@ impl BackupManifest {
|
|||||||
return Err(BackupError::corrupted("manifest has no digest slot"));
|
return Err(BackupError::corrupted("manifest has no digest slot"));
|
||||||
};
|
};
|
||||||
*slot = serde_json::Value::String(String::new());
|
*slot = serde_json::Value::String(String::new());
|
||||||
if Self::digest_of_canonical_value(&value, declared.algorithm)? != *declared {
|
if Self::digest_of_canonical_value(value, declared.algorithm)? != *declared {
|
||||||
return Err(BackupError::corrupted(
|
return Err(BackupError::corrupted(
|
||||||
"manifest digest mismatch: content does not match the sealed digest",
|
"manifest digest mismatch: content does not match the sealed digest",
|
||||||
));
|
));
|
||||||
@@ -511,8 +514,8 @@ impl BackupManifest {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn digest_of_canonical_value(value: &serde_json::Value, algorithm: DigestAlgorithm) -> Result<ContentDigest, BackupError> {
|
fn digest_of_canonical_value(value: serde_json::Value, algorithm: DigestAlgorithm) -> Result<ContentDigest, BackupError> {
|
||||||
let canonical = serde_json::to_vec(value)
|
let canonical = serde_json::to_vec(&canonicalize_value(value))
|
||||||
.map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
|
.map_err(|error| BackupError::corrupted(format!("manifest canonicalization failed: {error}")))?;
|
||||||
match algorithm {
|
match algorithm {
|
||||||
DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)),
|
DigestAlgorithm::Sha256 => Ok(ContentDigest::sha256_of(&canonical)),
|
||||||
@@ -701,6 +704,29 @@ fn map_serde_error(error: serde_json::Error) -> BackupError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rebuild a JSON value with object keys in bytewise-sorted order at every
|
||||||
|
/// nesting level (array element order is preserved).
|
||||||
|
///
|
||||||
|
/// `serde_json`'s map keeps keys sorted by default but preserves insertion
|
||||||
|
/// order when the `preserve_order` feature is unified into the build by any
|
||||||
|
/// other crate. Digest bytes must not depend on that, so the ordering is
|
||||||
|
/// imposed explicitly here instead of being inherited from the map type.
|
||||||
|
fn canonicalize_value(value: serde_json::Value) -> serde_json::Value {
|
||||||
|
match value {
|
||||||
|
serde_json::Value::Object(map) => {
|
||||||
|
let mut entries: Vec<(String, serde_json::Value)> = map.into_iter().collect();
|
||||||
|
entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||||
|
let mut sorted = serde_json::Map::with_capacity(entries.len());
|
||||||
|
for (key, entry) in entries {
|
||||||
|
sorted.insert(key, canonicalize_value(entry));
|
||||||
|
}
|
||||||
|
serde_json::Value::Object(sorted)
|
||||||
|
}
|
||||||
|
serde_json::Value::Array(items) => serde_json::Value::Array(items.into_iter().map(canonicalize_value).collect()),
|
||||||
|
other => other,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -1085,8 +1111,8 @@ mod tests {
|
|||||||
let mut value = serde_json::to_value(&sealed).expect("manifest should convert to a value");
|
let mut value = serde_json::to_value(&sealed).expect("manifest should convert to a value");
|
||||||
value["created_at"] = serde_json::Value::String("2026-07-30T00:00:00+00:00".to_string());
|
value["created_at"] = serde_json::Value::String("2026-07-30T00:00:00+00:00".to_string());
|
||||||
value["manifest_digest"]["hex"] = serde_json::Value::String(String::new());
|
value["manifest_digest"]["hex"] = serde_json::Value::String(String::new());
|
||||||
let canonical = serde_json::to_vec(&value).expect("canonical bytes");
|
let digest = BackupManifest::digest_of_canonical_value(value.clone(), DigestAlgorithm::Sha256)
|
||||||
let digest = ContentDigest::sha256_of(&canonical);
|
.expect("canonical digest should compute");
|
||||||
value["manifest_digest"]["hex"] = serde_json::Value::String(digest.hex);
|
value["manifest_digest"]["hex"] = serde_json::Value::String(digest.hex);
|
||||||
let bytes = serde_json::to_vec(&value).expect("manifest bytes");
|
let bytes = serde_json::to_vec(&value).expect("manifest bytes");
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user