diff --git a/Cargo.lock b/Cargo.lock index 60d3a33c78..19b1bcb97b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3592,6 +3592,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "pdfcraft-audit" +version = "0.6.0" +dependencies = [ + "pdfcraft-content", + "pdfcraft-cos", + "pdfcraft-edit", + "pdfcraft-model", + "thiserror 2.0.21", +] + [[package]] name = "pdfcraft-automation" version = "0.6.0" @@ -3614,6 +3625,7 @@ name = "pdfcraft-cli" version = "0.6.0" dependencies = [ "base64 0.22.1", + "pdfcraft-audit", "pdfcraft-automation", "pdfcraft-cos", "pdfcraft-engine", @@ -3697,6 +3709,7 @@ dependencies = [ "image", "pdfcraft-a11y", "pdfcraft-annot", + "pdfcraft-audit", "pdfcraft-compare", "pdfcraft-cos", "pdfcraft-create", diff --git a/Cargo.toml b/Cargo.toml index 1312fedc07..70a65b6b99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ pdfcraft-crypt = { path = "crates/crypt", version = "0.6.0" } pdfcraft-organize = { path = "crates/organize", version = "0.6.0" } pdfcraft-measure = { path = "crates/measure", version = "0.6.0" } pdfcraft-annot = { path = "crates/annot", version = "0.6.0" } +pdfcraft-audit = { path = "crates/audit", version = "0.6.0" } pdfcraft-fonts = { path = "crates/fonts", version = "0.6.0" } pdfcraft-model = { path = "crates/model", version = "0.6.0" } pdfcraft-content = { path = "crates/content", version = "0.6.0" } diff --git a/apps/pdfcraft-cli/Cargo.toml b/apps/pdfcraft-cli/Cargo.toml index 77dfe2f9fd..abe2657fbc 100644 --- a/apps/pdfcraft-cli/Cargo.toml +++ b/apps/pdfcraft-cli/Cargo.toml @@ -16,6 +16,7 @@ workspace = true pdfcraft-render = { workspace = true } serde_json = { workspace = true } pdfcraft-engine = { workspace = true } +pdfcraft-audit = { workspace = true } pdfcraft-cos = { workspace = true } pdfcraft-automation = { workspace = true } base64 = "0.22" diff --git a/apps/pdfcraft-cli/src/main.rs b/apps/pdfcraft-cli/src/main.rs index d8f0874bdc..f8861143c4 100644 --- a/apps/pdfcraft-cli/src/main.rs +++ b/apps/pdfcraft-cli/src/main.rs @@ -55,6 +55,7 @@ pdfcraft-cli combine … --out combined.pdf pdfcraft-cli extract --pages 1,3,5 --out out.pdf pdfcraft-cli split (--every N | --before 3,7) [--out-dir DIR] pdfcraft-cli check [--timeout 20] [--dpi 36] [--json out.json] +pdfcraft-cli audit [--password PW] find faux redactions (fails when found) pdfcraft-cli tools automation tools and their JSON Schemas pdfcraft-cli run [key=value …] [--root DIR] [--out image.png] pdfcraft-cli run --script steps.json [--root DIR] [{\"tool\": \"doc_open\", \"args\": {…}}, …] @@ -85,6 +86,7 @@ fn main() -> ExitCode { Some("split") => split(&args[1..]), Some("check") => check(&args[1..]), Some("check-one") => check_one(&args[1..]), + Some("audit") => audit(&args[1..]), Some("tools") => tools(), Some("run") => run(&args[1..]), Some("ui") => ui(&args[1..]), @@ -146,6 +148,28 @@ fn version() -> Result<(), CliError> { Ok(()) } +/// Audit a document for faux redactions. Prints the findings as JSON; fails when any are +/// found so scripts and CI can gate on it, and when the audit itself cannot run. +fn audit(args: &[String]) -> Result<(), CliError> { + let path = *positional(args).first().ok_or("audit: missing file")?; + let bytes = read(path)?; + let doc = pdfcraft_cos::Document::open_with_password(bytes, flag(args, "--password")).map_err(|e| format!("{path}: {e}"))?; + let findings = pdfcraft_audit::audit_redactions(&doc).map_err(|e| format!("{path}: {e}"))?; + let items: Vec = findings + .into_iter() + .map(|f| { + serde_json::json!({ + "page": f.page + 1, + "kind": f.kind.id(), + "rect": f.rect, + "covered_text": f.covered_text, + }) + }) + .collect(); + println!("{}", serde_json::json!({ "file": path, "findings": items, "count": items.len() })); + if items.is_empty() { Ok(()) } else { Err(format!("audit: {} faux redaction(s) found", items.len()).into()) } +} + fn flag<'a>(args: &'a [String], name: &str) -> Option<&'a str> { args.iter().position(|a| a == name).and_then(|i| args.get(i + 1)).map(String::as_str) } diff --git a/crates/audit/Cargo.toml b/crates/audit/Cargo.toml new file mode 100644 index 0000000000..5c55814218 --- /dev/null +++ b/crates/audit/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "pdfcraft-audit" +description = "Verification tooling: audit documents for faux redactions (opaque covers over extractable text)" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +pdfcraft-cos = { workspace = true } +pdfcraft-content = { workspace = true } +pdfcraft-edit = { workspace = true } +pdfcraft-model = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] diff --git a/crates/audit/README.md b/crates/audit/README.md new file mode 100644 index 0000000000..e13bb30b0d --- /dev/null +++ b/crates/audit/README.md @@ -0,0 +1,30 @@ +# pdfcraft-audit + +Layer L4 (may use `edit`): verification tooling, the faux-redaction audit. + +```rust +let findings: Vec = audit_redactions(&doc)?; // page, kind, rect, covered_text +``` + +It reports two kinds of finding: + +- **faux redaction**: an opaque filled rectangle painted *after* text it overlaps, dark enough + against the text's colour (WCAG contrast below 3) to hide it, while the text is still + extractable underneath; +- **unapplied mark**: a Redact annotation that was marked but never applied. + +A properly applied redaction removes the text, so it never flags. Page backgrounds, table +shading and highlights don't either: they're painted before the text, or light against it. + +Limits, by design and documented rather than hidden: + +- only `re` rectangles closed by a fill count as covers (not arbitrary paths, shadings or + patterns), and a translucent one (`/ca` below 1) is not flagged; +- covers inside form XObjects are found, but only text drawn directly in the page content is + checked under them; +- annotation rectangles are compared in unrotated user space, so `/Rotate` pages may misalign. + +It fails closed: an undecodable content stream, malformed `/Annots`, form XObjects nested deeper +than 32 or in a cycle, or a walk past the operator bound is an error, never "no findings". + +Surfaces: the `redact_audit` automation tool and `pdfcraft-cli`. diff --git a/crates/audit/src/cover.rs b/crates/audit/src/cover.rs new file mode 100644 index 0000000000..9da3fc7b68 --- /dev/null +++ b/crates/audit/src/cover.rs @@ -0,0 +1,242 @@ +//! Walk page content streams and collect the opaque filled rectangles they draw. +//! +//! This is the geometric half of the faux-redaction audit: a rectangle filled with an opaque +//! color (`re` + `f`) is the classic way a "redaction" is faked. Graphics state tracked here is +//! deliberately small: the CTM stack (`q`/`Q`/`cm`), the non-stroking color (`g`/`rg`/`k`) and +//! the non-stroking alpha from ExtGState (`gs` reading `/ca`). Form XObjects are followed +//! read-only with a depth cap. Anything fancier (arbitrary filled paths, shadings, patterns) is +//! out of scope and documented as such. +//! +//! Paint order: a page's content streams are one stream in pieces (ISO 32000-2 §7.8.2), so they +//! are parsed joined and walked once with a single graphics state — state set in an earlier +//! stream (a `cm`, a fill color) is still in force in the later ones. Each rectangle records the +//! position of the fill operator that painted it, as `(content-stream index, operator index in +//! the joined stream)`; rectangles inside a form XObject carry the `Do` operator's position. +//! Positions compare lexicographically in paint order. + +use pdfcraft_content::{Matrix, Op, Pieces, parse}; +use pdfcraft_cos::{Dict, Document, Object}; + +use crate::AuditError; + +/// Maximum form-XObject nesting followed. Deeper nesting (including a cycle) fails the audit: a +/// cover drawn below the cut could otherwise go unseen and the page be reported clean. +const MAX_DEPTH: usize = 32; +/// Bound on total operators walked per page, so a hostile stream cannot spin forever. +/// Hitting it fails the audit: a truncated walk must never report "clean". +const MAX_OPS: usize = 2_000_000; + +/// One opaque filled rectangle, in page user space. +#[derive(Clone, Debug, PartialEq)] +pub struct FilledRect { + /// `[x0, y0, x1, y1]`, normalized so `x0 <= x1` and `y0 <= y1`. + pub rect: [f64; 4], + /// The fill color as sRGB-ish `[r, g, b]` (CMYK is converted approximately). + pub rgb: [f64; 3], + /// Paint position: `(content-stream index, operator index in the joined stream)`. + /// Rectangles inside a form XObject carry the `Do` operator's position. + pub pos: (usize, usize), +} + +#[derive(Clone)] +struct Gs { + ctm: Matrix, + fill: [f64; 3], + alpha: f64, +} + +impl Default for Gs { + fn default() -> Self { + Gs { ctm: Matrix::IDENTITY, fill: [0.0, 0.0, 0.0], alpha: 1.0 } + } +} + +struct Walker<'a> { + doc: &'a Document, + pieces: &'a Pieces, + out: Vec, + ops_seen: usize, +} + +impl<'a> Walker<'a> { + fn bump(&mut self) -> Result<(), AuditError> { + self.ops_seen += 1; + if self.ops_seen > MAX_OPS { + return Err(AuditError::TooManyOps); + } + Ok(()) + } + + /// Walk one operator slice with the given graphics state. `at_do` is the `Do` position to + /// stamp on rectangles found inside a form XObject, or `None` at page level (where each + /// fill operator's own position is used). + fn walk_ops( + &mut self, + ops: &[Op], + gs: &mut Gs, + stack: &mut Vec, + resources: &Dict, + depth: usize, + at_do: Option<(usize, usize)>, + ) -> Result<(), AuditError> { + // A pending `re` rectangle, in the user space active when it was built. + let mut pending: Option<[f64; 4]> = None; + for (i, op) in ops.iter().enumerate() { + self.bump()?; + // Paint position of this operator: its own `(piece, joined index)` at page level, + // the enclosing `Do`'s position inside a form XObject. + let pos = match at_do { + Some(p) => p, + None => (self.pieces.pieces_of(op).0, i), + }; + match op.op.as_slice() { + b"q" => stack.push(gs.clone()), + b"Q" => { + if let Some(g) = stack.pop() { + *gs = g; + } + pending = None; + } + b"cm" => { + if let Some(m) = Matrix::from_operands(&op.operands) + && m.0.iter().all(|v| v.is_finite()) + { + gs.ctm = m.then(&gs.ctm); + } + pending = None; + } + b"g" => { + if let Some(v) = op.num(0).filter(|v| v.is_finite()) { + let v = v.clamp(0.0, 1.0); + gs.fill = [v, v, v]; + } + } + b"rg" => { + if let Some([r, g, b]) = op.nums::<3>() + && [r, g, b].iter().all(|v| v.is_finite()) + { + gs.fill = [r.clamp(0.0, 1.0), g.clamp(0.0, 1.0), b.clamp(0.0, 1.0)]; + } + } + b"k" => { + if let Some([c, m, y, k]) = op.nums::<4>() + && [c, m, y, k].iter().all(|v| v.is_finite()) + { + // Naive CMYK -> RGB; good enough to tell black from white. + let (c, m, y, k) = (c.clamp(0.0, 1.0), m.clamp(0.0, 1.0), y.clamp(0.0, 1.0), k.clamp(0.0, 1.0)); + gs.fill = [(1.0 - c) * (1.0 - k), (1.0 - m) * (1.0 - k), (1.0 - y) * (1.0 - k)]; + } + } + b"gs" => { + if let Some(name) = op.name(0) { + gs.alpha = ext_alpha(self.doc, resources, name); + } + } + b"re" => { + pending = op.nums::<4>().and_then(|[x, y, w, h]| { + if [x, y, w, h].iter().all(|v| v.is_finite()) && w != 0.0 && h != 0.0 { + let r = gs.ctm.bbox([x.min(x + w), y.min(y + h), x.max(x + w), y.max(y + h)]); + (r[2] > r[0] && r[3] > r[1]).then_some(r) + } else { + None + } + }); + } + b"f" | b"F" | b"f*" | b"B" | b"B*" => { + if let Some(r) = pending.take() + && gs.alpha >= 1.0 + { + self.out.push(FilledRect { rect: r, rgb: gs.fill, pos }); + } + // Any other path construction clears the pending rectangle. + } + b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"S" | b"s" | b"n" | b"W" | b"W*" => { + pending = None; + } + b"Do" => { + if let Some(name) = op.name(0) { + if depth >= MAX_DEPTH { + return Err(AuditError::TooDeep); + } + self.walk_xobject(name, gs, resources, depth, pos)?; + } + pending = None; + } + _ => {} + } + } + Ok(()) + } + + /// Follow a form XObject read-only, composing its matrix onto the current CTM. State + /// changes inside never leak out; every rectangle found is stamped with the `Do` + /// position `pos`. + fn walk_xobject(&mut self, name: &[u8], gs: &Gs, resources: &Dict, depth: usize, pos: (usize, usize)) -> Result<(), AuditError> { + let xobjects = resources.get(b"XObject").map(|o| self.doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default(); + let Some(entry) = xobjects.get(name) else { return Ok(()) }; + let resolved = self.doc.resolve(entry); + let Object::Stream(stream) = &*resolved else { return Ok(()) }; + let dict = &stream.dict; + if dict.name(b"Subtype") != Some(b"Form") { + return Ok(()); + } + let matrix = dict + .get(b"Matrix") + .map(|o| self.doc.resolve(o)) + .and_then(|o| o.as_array().map(|a| a.to_vec())) + .and_then(|a| { + let v: Vec = a.iter().filter_map(|o| self.doc.resolve(o).as_f64()).collect(); + (v.len() == 6 && v.iter().all(|x| x.is_finite())).then(|| Matrix([v[0], v[1], v[2], v[3], v[4], v[5]])) + }) + .unwrap_or(Matrix::IDENTITY); + let sub_resources = dict.get(b"Resources").map(|o| self.doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default(); + // A form stream that cannot be decoded fails the audit: it might hide a cover. + let data = stream.decoded_strict()?; + let ops = parse(&data).ops; + // The XObject matrix maps form space into the user space active at `Do`. + let mut sub_gs = gs.clone(); + sub_gs.ctm = matrix.then(&gs.ctm); + let mut sub_stack = Vec::new(); + self.walk_ops(&ops, &mut sub_gs, &mut sub_stack, &sub_resources, depth + 1, Some(pos)) + } +} + +/// The non-stroking alpha (`/ca`) of a named ExtGState, defaulting to opaque. +fn ext_alpha(doc: &Document, resources: &Dict, name: &[u8]) -> f64 { + let alpha = resources + .get(b"ExtGState") + .map(|o| doc.resolve(o)) + .and_then(|o| o.as_dict().cloned()) + .and_then(|d| d.get(name).map(|o| doc.resolve(o))) + .and_then(|o| o.as_dict().cloned()) + .and_then(|d| d.get(b"ca").map(|o| doc.resolve(o)).and_then(|o| o.as_f64())); + alpha.filter(|a| a.is_finite()).map(|a| a.clamp(0.0, 1.0)).unwrap_or(1.0) +} + +/// Every opaque filled rectangle drawn by a page's content (and its form XObjects), with paint +/// positions. A page's content streams are walked once, joined, with one graphics state, so +/// state set in an earlier stream is still in force in the later ones. Any content stream that +/// cannot be decoded is an audit failure, never an empty result. +pub fn filled_rects(doc: &Document, page: &Dict) -> Result, AuditError> { + let resources = page.get(b"Resources").map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default(); + let contents: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*doc.resolve(c) { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut streams = Vec::with_capacity(contents.len()); + for c in &contents { + if let Object::Stream(s) = &*doc.resolve(c) { + streams.push(s.decoded_strict()?); + } + } + let pieces = Pieces::join(&streams); + let parsed = pieces.parse(); + let mut w = Walker { doc, pieces: &pieces, out: Vec::new(), ops_seen: 0 }; + let mut gs = Gs::default(); + let mut stack = Vec::new(); + w.walk_ops(&parsed.ops, &mut gs, &mut stack, &resources, 0, None)?; + Ok(w.out) +} diff --git a/crates/audit/src/lib.rs b/crates/audit/src/lib.rs new file mode 100644 index 0000000000..ad26c25596 --- /dev/null +++ b/crates/audit/src/lib.rs @@ -0,0 +1,192 @@ +//! Verification tooling: audit a document for faux redactions. +//! +//! [`audit_redactions`] finds *faux redactions* — opaque filled rectangles drawn over text +//! that is still extractable underneath — and redaction annotations that were marked but +//! never applied. A properly applied redaction removes the text, so it never flags. +//! +//! A rectangle counts as a cover only when it was painted strictly *after* the text it +//! overlaps, and only when it is dark enough against the text's color to make the text +//! unreadable (WCAG contrast below 3.0). Page backgrounds, table shading and highlight +//! marks are therefore not flagged. +//! +//! Detection limits (documented, not hidden): only `re` rectangles closed by a fill operator +//! are considered — arbitrary filled paths, shadings and patterns are not; a rectangle drawn +//! with a translucent ExtGState (`/ca < 1`) is not flagged; annotation rectangles are compared +//! in unrotated user space, so pages with `/Rotate` may misalign; covers inside form XObjects +//! are found, but only text drawn directly in the page content is checked under them, so text +//! inside a form XObject is not audited. +//! +//! The audit fails closed: any content that cannot be read (an undecodable content stream, +//! malformed annotations, form XObjects nested deeper than 32 or in a cycle) is an error, never +//! an empty finding list. + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use pdfcraft_cos::{Dict, Document}; +use pdfcraft_edit::TextLine; + +mod cover; + +#[cfg(test)] +mod tests; + +/// What the audit found. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FindingKind { + /// An opaque filled rectangle covers text that is still extractable. + CoveredText, + /// A `/Redact` annotation was placed but never applied: the text is untouched. + UnappliedMark, +} + +impl FindingKind { + pub fn id(self) -> &'static str { + match self { + FindingKind::CoveredText => "covered-text", + FindingKind::UnappliedMark => "unapplied-mark", + } + } +} + +/// One audit finding. +#[derive(Clone, Debug, PartialEq)] +pub struct AuditFinding { + /// 0-based page index. + pub page: usize, + pub kind: FindingKind, + /// The covering rectangle (or the mark's rectangle), in page user space. + pub rect: [f64; 4], + /// The extractable text the cover hides (possibly empty when nothing decodable remains). + pub covered_text: String, +} + +#[derive(Debug, thiserror::Error)] +pub enum AuditError { + #[error("page {0}: {1}")] + Text(usize, String), + #[error("content walk exceeded the operator bound")] + TooManyOps, + #[error("form XObjects nest deeper than the audit follows")] + TooDeep, + #[error("page has a malformed /Annots entry")] + BadAnnots, + #[error(transparent)] + Cos(#[from] pdfcraft_cos::CosError), + #[error(transparent)] + Edit(#[from] pdfcraft_edit::EditError), +} + +/// Fraction of a text line's area that must lie under a cover to count as hidden. +const COVER_FRACTION: f64 = 0.4; +/// WCAG contrast ratio below which the text is unreadable against the cover. +const COVER_CONTRAST: f64 = 3.0; + +/// Audit every page for faux redactions and unapplied redaction marks. Fails closed: any +/// content that cannot be decoded is returned as an error, never as an empty finding list. +pub fn audit_redactions(doc: &Document) -> Result, AuditError> { + let mut out = Vec::new(); + for (n, page) in pdfcraft_model::pages(doc).iter().enumerate() { + let lines = pdfcraft_edit::text_lines(doc, n).map_err(|e| AuditError::Text(n, e.to_string()))?; + // 1. Opaque filled rectangles over extractable text. + for r in cover::filled_rects(doc, &page.dict)? { + let mut covered = String::new(); + for line in &lines { + if is_cover(&r, line) { + if !covered.is_empty() { + covered.push(' '); + } + covered.push_str(line.text.trim()); + } + } + if !covered.trim().is_empty() { + out.push(AuditFinding { page: n, kind: FindingKind::CoveredText, rect: r.rect, covered_text: covered }); + } + } + // 2. Redact annotations that were never applied. + for annot in annots(doc, &page.dict)? { + let mut covered = String::new(); + for line in &lines { + if covers(&annot, &line.rect) { + if !covered.is_empty() { + covered.push(' '); + } + covered.push_str(line.text.trim()); + } + } + if !covered.trim().is_empty() { + out.push(AuditFinding { page: n, kind: FindingKind::UnappliedMark, rect: annot, covered_text: covered }); + } + } + } + Ok(out) +} + +/// A filled rectangle is a faux-redaction cover for a text line iff it was painted strictly +/// after the line, overlaps it substantially, and is dark enough against the line's color to +/// make the text unreadable. Light covers (table shading, highlights) over dark text fail the +/// contrast check and are not flagged. +fn is_cover(rect: &cover::FilledRect, line: &TextLine) -> bool { + rect.pos > line.paint_pos() && covers(&rect.rect, &line.rect) && contrast(line.color, rect.rgb) < COVER_CONTRAST +} + +/// Does `cover` hide a substantial part of `line`? +fn covers(cover: &[f64; 4], line: &[f64; 4]) -> bool { + let ix0 = cover[0].max(line[0]); + let iy0 = cover[1].max(line[1]); + let ix1 = cover[2].min(line[2]); + let iy1 = cover[3].min(line[3]); + if ix1 <= ix0 || iy1 <= iy0 { + return false; + } + let line_area = (line[2] - line[0]).max(0.0) * (line[3] - line[1]).max(0.0); + if line_area <= 0.0 { + return false; + } + (ix1 - ix0) * (iy1 - iy0) / line_area >= COVER_FRACTION +} + +/// WCAG 2.x relative luminance of an sRGB triple. +fn luminance(rgb: [f64; 3]) -> f64 { + let lin = rgb.map(|c| { + let c = c.clamp(0.0, 1.0); + if c <= 0.03928 { c / 12.92 } else { ((c + 0.055) / 1.055).powf(2.4) } + }); + 0.2126 * lin[0] + 0.7152 * lin[1] + 0.0722 * lin[2] +} + +/// WCAG contrast ratio between two sRGB colors: 1.0 (identical) to 21.0 (black on white). +fn contrast(a: [f64; 3], b: [f64; 3]) -> f64 { + let (l1, l2) = (luminance(a), luminance(b)); + let (hi, lo) = if l1 > l2 { (l1, l2) } else { (l2, l1) }; + (hi + 0.05) / (lo + 0.05) +} + +/// Rectangles of `/Redact` annotations on a page (unrotated user space). A missing `/Annots` +/// means no marks; a malformed one fails the audit rather than silently passing. +fn annots(doc: &Document, page: &Dict) -> Result, AuditError> { + let Some(a) = page.get(b"Annots") else { return Ok(Vec::new()) }; + let resolved = doc.resolve(a); + let items: Vec = match &*resolved { + pdfcraft_cos::Object::Array(items) => items.clone(), + // A missing /Annots means no marks; a malformed one fails the audit. + _ => return Err(AuditError::BadAnnots), + }; + let mut out = Vec::new(); + for a in &items { + let dict = match &*doc.resolve(a) { + pdfcraft_cos::Object::Dict(d) => d.clone(), + _ => continue, + }; + if dict.name(b"Subtype") != Some(b"Redact") { + continue; + } + let v: Vec = match dict.get(b"Rect") { + None => continue, + Some(r) => doc.resolve(r).as_array().map(|arr| arr.iter().filter_map(|o| doc.resolve(o).as_f64()).collect()).unwrap_or_default(), + }; + if v.len() == 4 && v.iter().all(|x| x.is_finite()) { + out.push([v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]); + } + } + Ok(out) +} diff --git a/crates/audit/src/tests.rs b/crates/audit/src/tests.rs new file mode 100644 index 0000000000..edeb2f95df --- /dev/null +++ b/crates/audit/src/tests.rs @@ -0,0 +1,215 @@ +//! Tests build their own tiny PDFs in code (contributor-original fixtures): no binary files, +//! no Adobe material, exact xref offsets computed by the builder below. + +use std::sync::Arc; + +use pdfcraft_cos::Document; + +use crate::{FindingKind, audit_redactions}; + +/// Minimal PDF assembler: objects are numbered 1..=n in order. +struct Builder { + objects: Vec>, +} + +impl Builder { + fn new() -> Self { + Builder { objects: Vec::new() } + } + + fn add(&mut self, body: &str) -> usize { + self.objects.push(body.as_bytes().to_vec()); + self.objects.len() + } + + fn add_stream(&mut self, data: &[u8]) -> usize { + self.add_stream_with(data, "") + } + + fn add_stream_with(&mut self, data: &[u8], dict_extra: &str) -> usize { + let mut body = format!("<< /Length {} {dict_extra} >>\nstream\n", data.len()).into_bytes(); + body.extend_from_slice(data); + body.extend_from_slice(b"\nendstream"); + self.objects.push(body); + self.objects.len() + } + + fn build(self) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, obj) in self.objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(obj); + out.extend_from_slice(b"\nendobj\n"); + } + let xref_at = out.len(); + let n = self.objects.len() + 1; + out.extend_from_slice(format!("xref\n0 {n}\n").as_bytes()); + out.extend_from_slice(b"0000000000 65535 f \n"); + for off in offsets { + out.extend_from_slice(format!("{off:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(b"trailer\n<< "); + out.extend_from_slice(format!("/Size {n} /Root 1 0 R").as_bytes()); + out.extend_from_slice(b" >>\nstartxref\n"); + out.extend_from_slice(format!("{xref_at}\n").as_bytes()); + out.extend_from_slice(b"%%EOF\n"); + out + } +} + +const FONT: &str = "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"; + +/// One page with `content` as its stream; `page_extra` is appended inside the page dict. +fn one_page(content: &[u8], page_extra: &str) -> Vec { + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + let stream_num = 4; + b.add(&format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {stream_num} 0 R /Resources << /Font << /F1 5 0 R >> >> {page_extra} >>" + )); + b.add_stream(content); + b.add(FONT); + b.build() +} + +/// One page with two content streams (`/Contents [4 0 R 5 0 R]`). +fn two_streams(s1: &[u8], s2: &[u8]) -> Vec { + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents [4 0 R 5 0 R] /Resources << /Font << /F1 6 0 R >> >> >>"); + b.add_stream(s1); + b.add_stream(s2); + b.add(FONT); + b.build() +} + +/// One page whose content stream claims `/FlateDecode` but holds garbage: decoding must fail. +fn bad_filter_page() -> Vec { + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"); + b.add_stream_with(b"this is not deflated data", "/Filter /FlateDecode"); + b.add(FONT); + b.build() +} + +fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).expect("test pdf must open") +} + +#[test] +fn faux_redaction_is_found() { + let content = b"BT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n0 g\n100 688 220 32 re f\n"; + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + assert_eq!(findings.len(), 1, "{findings:?}"); + let f = &findings[0]; + assert_eq!(f.page, 0); + assert_eq!(f.kind, FindingKind::CoveredText); + assert!(f.covered_text.contains("SECRET"), "covered_text was {:?}", f.covered_text); +} + +#[test] +fn clean_document_has_no_findings() { + let content = b"BT /F1 24 Tf 100 700 Td (nothing hidden here) Tj ET\n"; + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn black_box_without_text_is_not_a_finding() { + // A filled rectangle with no text under it is decoration, not a faux redaction. + let content = b"BT /F1 24 Tf 100 100 Td (far away) Tj ET\n0 g\n400 600 100 40 re f\n"; + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn translucent_cover_is_not_flagged() { + // A rectangle drawn with a translucent ExtGState does not hide the text. + let content = b"/GS1 gs\nBT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n0 g\n100 688 220 32 re f\n"; + let page_extra = "/Resources << /Font << /F1 5 0 R >> /ExtGState << /GS1 6 0 R >> >>"; + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add(&format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R {page_extra} >>")); + b.add_stream(content); + b.add(FONT); + b.add("<< /Type /ExtGState /ca 0.5 >>"); + let findings = audit_redactions(&open(b.build())).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn unapplied_redact_mark_is_found() { + let content = b"BT /F1 24 Tf 100 700 Td (DO NOT SHARE) Tj ET\n"; + let page_extra = "/Annots [6 0 R]"; + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add(&format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> {page_extra} >>")); + b.add_stream(content); + b.add(FONT); + b.add("<< /Type /Annot /Subtype /Redact /Rect [95 685 305 725] /C [1 0 0] >>"); + let findings = audit_redactions(&open(b.build())).expect("audit must run"); + assert_eq!(findings.len(), 1, "{findings:?}"); + assert_eq!(findings[0].kind, FindingKind::UnappliedMark); + assert!(findings[0].covered_text.contains("DO NOT SHARE")); +} + +#[test] +fn background_before_text_is_not_flagged() { + // An opaque dark rectangle painted *before* the text is a background, not a cover. + let content = b"0 g\n100 688 220 32 re f\nBT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n"; + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn light_shading_after_text_is_not_flagged() { + // Light table-style shading painted after the text leaves it readable: not a cover. + let content = b"BT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n0.9 g\n100 688 220 32 re f\n"; + let findings = audit_redactions(&open(one_page(content, ""))).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn graphics_state_carries_across_streams() { + // The first stream sets a white fill after the text; the second stream's rectangle over + // the text is therefore white, not the default black: no finding. Without the carry fix + // the rectangle would default to black and flag. + let s1 = b"BT /F1 24 Tf 100 700 Td (SECRET DATA 123) Tj ET\n1 g\n"; + let s2 = b"100 688 220 32 re f\n"; + let findings = audit_redactions(&open(two_streams(s1, s2))).expect("audit must run"); + assert!(findings.is_empty(), "{findings:?}"); +} + +#[test] +fn audit_fails_closed_on_undecodable_stream() { + // A content stream that cannot be decoded is an audit error, never an empty finding list. + let err = audit_redactions(&open(bad_filter_page())).expect_err("audit must fail closed"); + let msg = err.to_string(); + assert!(msg.contains("decoded"), "unexpected error: {msg}"); +} + +/// A form XObject that draws itself nests without end. The walk can't follow it to the bottom, so +/// the audit fails rather than report the page clean with a possible cover left unseen. +#[test] +fn self_drawing_form_fails_the_audit() { + let mut b = Builder::new(); + b.add("<< /Type /Catalog /Pages 2 0 R >>"); + b.add("<< /Type /Pages /Kids [3 0 R] /Count 1 >>"); + b.add("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 6 0 R >> /XObject << /X 5 0 R >> >> >>"); + b.add_stream(b"BT /F1 24 Tf 100 700 Td (SECRET) Tj ET\n/X Do\n"); + b.add_stream_with( + b"0 g\n100 688 220 32 re f\n/X Do\n", + "/Type /XObject /Subtype /Form /BBox [0 0 612 792] /Resources << /XObject << /X 5 0 R >> >>", + ); + b.add(FONT); + let result = audit_redactions(&open(b.build())); + assert!(matches!(result, Err(crate::AuditError::TooDeep)), "{result:?}"); +} diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 1e2c872ffd..71ab531f56 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -587,6 +587,7 @@ impl Automation { "content_delete" => self.content_delete(&a)?, "doc_print" => self.doc_print(&a)?, "doc_remove_hidden" => self.doc_remove_hidden(&a)?, + "doc_audit_redactions" => self.doc_audit_redactions(&a)?, "fill_sign_add" => self.fill_sign_add(&a)?, "fill_sign_date_format" => self.fill_sign_date_format(&a)?, "measure_distance" => self.measurement_add(&a, pdfcraft_engine::measure::Kind::Distance)?, diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index ec8f232dbc..44baf5e6f8 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -163,6 +163,22 @@ impl Automation { Ok(json!({ "total": total, "categories": items })) } + pub(crate) fn doc_audit_redactions(&self, a: &Args) -> Result { + let findings = self.doc(a)?.audit_redactions().map_err(ToolError::Failed)?; + let items: Vec = findings + .into_iter() + .map(|f| { + json!({ + "page": f.page + 1, + "kind": f.kind.id(), + "rect": f.rect, + "covered_text": f.covered_text, + }) + }) + .collect(); + Ok(json!({ "findings": items, "count": items.len() })) + } + pub(crate) fn doc_remove_hidden(&mut self, a: &Args) -> Result { let edit = match a.get("categories") { None => Edit::Sanitize, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index bf92100636..f2d947d480 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -447,6 +447,13 @@ pub fn tools() -> Vec { }), &["doc"], )), + t( + "doc_audit_redactions", + "Audit faux redactions", + "Find faux redactions: opaque filled rectangles painted over text that is still extractable, and /Redact annotations that were marked but never applied. Paint-order and colour-aware: a rectangle counts only when painted after the text and dark enough to hide it (page backgrounds and light shading are not flagged). Each finding reports the 1-based page, the kind (covered-text or unapplied-mark), the rectangle, and the hidden text. Fails closed: unreadable content is an error, never a clean bill. A properly applied redaction removes the text and never flags.", + ) + .ro() + .with(schema(json!({ "doc": doc() }), &["doc"])), t("printers", "List printers", "The printers the system's print spooler knows (CUPS on macOS and Linux), with the default marked.").ro().with(schema(json!({}), &[])), t("printer_options", "List printer options", "A printer driver's own job options (CUPS: from its PPD), such as the paper tray, paper type or finishing: key, label, group, default and choices. Pass chosen values to doc_print as options. Empty on Windows, where the driver's preferences window holds them.") .ro() diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index a1e3133f94..d643af351d 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -2366,6 +2366,50 @@ fn removing_hidden_information_through_tools() { assert!(matches!(a.call("doc_remove_hidden", &json!({ "doc": doc, "categories": ["nonsense"] })), Err(ToolError::InvalidArgs(_)))); } +/// A one-page PDF whose text "TOP SECRET" sits under an opaque black rectangle. +fn faux_redacted_fixture() -> Vec { + let body = "BT /F1 24 Tf 100 700 Td (TOP SECRET) Tj ET\n0 g\n90 688 220 32 re f"; + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [4 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 5 0 R /Resources << /Font << /F1 3 0 R >> >> >>".to_string().into_bytes(), + format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len()).into_bytes(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +#[test] +fn auditing_faux_redactions_through_tools() { + let dir = workdir("audit"); + std::fs::write(dir.join("redacted.pdf"), faux_redacted_fixture()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "redacted.pdf" }))["doc"].as_u64().unwrap(); + let r = ok(&mut a, "doc_audit_redactions", json!({ "doc": doc })); + assert_eq!(r["count"], 1, "{r}"); + let f = &r["findings"][0]; + assert_eq!(f["page"], 1); + assert_eq!(f["kind"], "covered-text"); + assert!(f["covered_text"].as_str().unwrap().contains("TOP SECRET"), "{f}"); + // A clean document audits clean. + let clean = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut a, "doc_audit_redactions", json!({ "doc": clean }))["count"], 0); +} + #[test] fn printing_through_tools() { let dir = workdir("print"); diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index b3c45ff113..e84ab1ad26 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -129,6 +129,14 @@ impl TextLine { pub fn origin_baseline(&self) -> f64 { self.origin.baseline } + + /// Paint position of the line's last text-showing operator: `(content-stream index, + /// operator index in the page's joined stream)`. Lines paint in order of this tuple. + /// (Lines from `text_lines` never come from form XObjects, so `stream` is always a + /// page content-stream index and `ops` are indices into the joined operator list.) + pub fn paint_pos(&self) -> (usize, usize) { + (self.stream, self.ops.iter().max().copied().unwrap_or(0)) + } } /// What replacing a line did. diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 4e754f7e32..eecbb35c4b 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -18,6 +18,7 @@ pdfcraft-annot = { workspace = true } pdfcraft-forms = { workspace = true } pdfcraft-xfa = { workspace = true } pdfcraft-edit = { workspace = true } +pdfcraft-audit = { workspace = true } pdfcraft-redact = { workspace = true } pdfcraft-print = { workspace = true } pdfcraft-xfdf = { workspace = true } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 23b712bba6..c2182348dc 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -514,6 +514,14 @@ impl Document { self.editor.as_ref().map(|e| pdfcraft_redact::sanitize::scan(&e.cos)).unwrap_or_default() } + /// Verification: faux redactions (opaque covers over extractable text) and redaction + /// marks that were never applied. Fails closed: any read error is returned, never an + /// empty list. + pub fn audit_redactions(&self) -> Result, String> { + let editor = self.editor.as_ref().ok_or("the document can't be read")?; + pdfcraft_audit::audit_redactions(&editor.cos).map_err(|e| e.to_string()) + } + /// Every page's media, crop, bleed, trim and art boxes (user space), for Set Page Boxes. pub fn page_boxes(&self) -> Vec<[[f64; 4]; 5]> { self.editor.as_ref().and_then(|e| pdfcraft_organize::page_boxes(&e.cos).ok()).unwrap_or_default() diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 55d2b2e77e..4786c27566 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5761,6 +5761,18 @@ tools = ["doc_remove_hidden"] evidence = ["crates/redact/src/tests.rs::hidden_information_is_counted_and_removed", "crates/ui-egui/tests/redact.rs::removing_hidden_information_and_sanitizing", "crates/automation/tests/automation.rs::removing_hidden_information_through_tools"] notes = "every category, then a full rewrite on save" +[[feature]] +id = "protect.audit-faux-redactions" +title = "Audit: detect faux redactions (opaque covers over extractable text, unapplied marks)" +area = "G" +tier = "P2" +milestone = "M8" +status = "partial" +commands = [] +tools = ["doc_audit_redactions"] +evidence = ["crates/audit/src/tests.rs::faux_redaction_is_found", "crates/audit/src/tests.rs::background_before_text_is_not_flagged", "crates/audit/src/tests.rs::light_shading_after_text_is_not_flagged", "crates/audit/src/tests.rs::graphics_state_carries_across_streams", "crates/audit/src/tests.rs::audit_fails_closed_on_undecodable_stream", "crates/audit/src/tests.rs::unapplied_redact_mark_is_found", "crates/automation/tests/automation.rs::auditing_faux_redactions_through_tools"] +notes = "New pdfcraft-audit crate. A filled rectangle (re + fill, opaque) counts as a cover only when painted after the text it overlaps and dark enough to hide it (WCAG contrast < 3.0); page backgrounds, table shading and highlights are not flagged. Graphics state carries across a page's content streams. Fails closed: undecodable content is an error, never a clean bill. Also flags /Redact annotations never applied. CLI: pdfcraft-cli audit (fails when findings exist)." + [[feature]] id = "protect.remove-hidden-on-close" title = "Remove hidden information when closing or sending" diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 72a199c10d..1bda1dc4d7 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -55,6 +55,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("organize", Class::Layer(4)), ("security", Class::Layer(4)), ("redact", Class::Layer(4)), + ("audit", Class::Layer(4)), ("xfdf", Class::Layer(4)), ("sign", Class::Layer(4)), ("ocr", Class::Layer(4)), @@ -96,6 +97,7 @@ pub const STANDALONE_DEPS: &[(&str, &[&str])] = &[("cos", &["filters", "crypt", /// Allowed same-layer edges (architecture §3 rule 1). `forms → js` is deliberately absent. pub const SIDEWAYS: &[(&str, &str)] = &[ ("redact", "edit"), + ("audit", "edit"), ("a11y", "preflight"), ("prepress", "preflight"), ("export", "ocr"),