diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ac7bdeaf4..388b44f032 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,12 +55,11 @@ jobs: sudo apt-get install -y --no-install-recommends \ libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev \ libgl1-mesa-dev libgtk-3-dev \ - qpdf + qpdf poppler-utils # The independent oracles (crates/oracle) are only worth having when they run. With this # set, a missing one fails the job instead of printing a skip notice that reads as a pass. - # Only the tools the job actually uses are installed above; a test that needs another one - # (Poppler, for the pixel comparison in crates/organize) stays `#[ignore]`d until then. + # Only the tools the job actually uses are installed above. - name: Require the oracles (Linux) if: runner.os == 'Linux' run: echo "PDFCRAFT_ORACLES_REQUIRED=1" >> "$GITHUB_ENV" diff --git a/crates/oracle/README.md b/crates/oracle/README.md index d02f5b5f9f..c94e080638 100644 --- a/crates/oracle/README.md +++ b/crates/oracle/README.md @@ -62,9 +62,28 @@ test result: FAILED. 1 failed `or_skip!` expands to a `return`, so it belongs in a test returning `()`; a test returning `Result` should match on the probe directly. +## Comparing pixels (`pixels`) + +Rendering needs more than "did the tool run". A raw pixel-for-pixel difference is useless here: +two correct renderers disagree on every glyph edge, so a strict comparison calls a perfect page +badly wrong and tells you nothing about whether a shading is missing. + +`pixels::compare` counts a pixel as differing only when it differs from **every** pixel in the +other image's 3x3 neighbourhood by more than a tolerance. A one-pixel shift or a differently +weighted edge finds a near match next door and is forgiven; a glyph that is not there, or a region +painted the wrong colour, has no near match and is counted. The test runs in both directions and +reports the worse result, so neither argument order flatters the blurrier renderer. + +`decode_pgm` reads what `pdftoppm -gray` writes; `decode_pam_gray` reads what +`pdfcraft-cli render --out x.pam` writes, compositing premultiplied alpha over white first so a +transparent margin is not scored as black. + +`cargo xtask render-oracle` drives it over the corpus. It reports; it does not gate. + ## Still to do -- CI installs only `qpdf`, the one tool the test job uses. The Poppler pixel comparison in - `crates/organize` stays `#[ignore]`d until a change turns it on and verifies it there. +- Acrobat itself as the reference, rather than other open renderers. - Oracles that are not external processes (hayro, `lopdf`) are ordinary dev-dependencies and are not routed through this crate. +- Two renderers sometimes round the same page box to different pixel sizes, and those files are + counted as unmeasurable rather than compared. diff --git a/crates/oracle/src/lib.rs b/crates/oracle/src/lib.rs index 440fb6e0f7..e9e7ae372e 100644 --- a/crates/oracle/src/lib.rs +++ b/crates/oracle/src/lib.rs @@ -23,6 +23,8 @@ //! This crate is test scaffolding (`Class::Testkit` in `xtask/src/layers.rs`): a dev-dependency //! only, never compiled into the product. +pub mod pixels; + use std::fmt; use std::process::Command; diff --git a/crates/oracle/src/pixels.rs b/crates/oracle/src/pixels.rs new file mode 100644 index 0000000000..7365e37eaf --- /dev/null +++ b/crates/oracle/src/pixels.rs @@ -0,0 +1,428 @@ +//! Comparing a rendered page against an independent renderer's pixels. +//! +//! A raw pixel-for-pixel difference is useless for this. Two correct renderers disagree on every +//! glyph edge — antialiasing coverage, hinting, and where a subpixel-positioned stem lands — so a +//! strict comparison reports a page as badly wrong when it is perfect, and the number it produces +//! tells you nothing about whether a shading is missing or a glyph is the wrong one. +//! +//! [`compare`] uses the usual remedy: a pixel counts as differing only when it differs from +//! **every** pixel in the other image's 3×3 neighbourhood by more than `tolerance`. A one-pixel +//! shift in a stem, or a differently-weighted edge, finds a near match next door and is forgiven. +//! A glyph that is not there, a shading that was not drawn, or a region painted the wrong colour +//! has no near match anywhere nearby and is counted. +//! +//! The test is run in both directions and the worse result reported, because the one-directional +//! form is asymmetric in a way that flatters whichever image is blurrier: thin content missing +//! from A is forgiven when scanning A against B's neighbourhood if B's stroke happens to be wide. + +/// A single-channel (grayscale) image borrowed for comparison. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Gray<'a> { + width: usize, + height: usize, + pixels: &'a [u8], +} + +impl<'a> Gray<'a> { + /// Borrow `pixels` as a `width` × `height` image, or `None` when the buffer is not exactly + /// that size — a truncated decode must not be compared as if it were a short page. + pub fn new(width: usize, height: usize, pixels: &'a [u8]) -> Option { + (width.checked_mul(height)? == pixels.len()).then_some(Self { width, height, pixels }) + } + + pub fn width(&self) -> usize { + self.width + } + + pub fn height(&self) -> usize { + self.height + } + + /// The pixel at `(x, y)`, or `None` outside the image. + pub fn get(&self, x: usize, y: usize) -> Option { + if x >= self.width || y >= self.height { + return None; + } + self.pixels.get(y.checked_mul(self.width)?.checked_add(x)?).copied() + } +} + +/// Why two images could not be compared. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SizeMismatch { + pub a: (usize, usize), + pub b: (usize, usize), +} + +impl std::fmt::Display for SizeMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cannot compare a {}x{} image with a {}x{} one; render both at the same scale", self.a.0, self.a.1, self.b.0, self.b.1) + } +} + +/// How far apart two renderings are. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Diff { + /// Pixels with no near match in the other image's neighbourhood, in the worse direction. + pub differing: usize, + /// Pixels in the image. + pub total: usize, + /// The largest single-pixel difference found, before the neighbourhood is considered. A high + /// value with a low [`Diff::fraction`] is the signature of an edge-only disagreement. + pub max_delta: u8, +} + +impl Diff { + /// The share of pixels with no near match, in `0.0..=1.0`. Zero for an empty image: nothing + /// disagreed because there was nothing to disagree about. + pub fn fraction(&self) -> f64 { + if self.total == 0 { + return 0.0; + } + // total is non-zero, and differing <= total, so this is finite and in 0..=1. + self.differing as f64 / self.total as f64 + } + + /// `fraction` as a percentage, for reports. + pub fn percent(&self) -> f64 { + self.fraction() * 100.0 + } +} + +/// Compare two grayscale renderings of the same page at the same size. +/// +/// `tolerance` is the per-channel difference tolerated outright; 8 or so absorbs ordinary +/// antialiasing weight differences without hiding a wrong colour. +pub fn compare(a: Gray<'_>, b: Gray<'_>, tolerance: u8) -> Result { + if a.width != b.width || a.height != b.height { + return Err(SizeMismatch { a: (a.width, a.height), b: (b.width, b.height) }); + } + let max_delta = a.pixels.iter().zip(b.pixels).map(|(&p, &q)| p.abs_diff(q)).max().unwrap_or(0); + let forward = unmatched(a, b, tolerance); + let backward = unmatched(b, a, tolerance); + Ok(Diff { differing: forward.max(backward), total: a.pixels.len(), max_delta }) +} + +/// Pixels of `a` with no pixel within `tolerance` anywhere in `b`'s surrounding 3x3. +fn unmatched(a: Gray<'_>, b: Gray<'_>, tolerance: u8) -> usize { + let mut count = 0usize; + for y in 0..a.height { + for x in 0..a.width { + let Some(p) = a.get(x, y) else { continue }; + let mut matched = false; + // Saturating bounds keep the window inside the image at the edges; `get` rejects the + // rest, so no index is ever derived from an unchecked subtraction. + for ny in y.saturating_sub(1)..=y.saturating_add(1) { + for nx in x.saturating_sub(1)..=x.saturating_add(1) { + if b.get(nx, ny).is_some_and(|q| p.abs_diff(q) <= tolerance) { + matched = true; + break; + } + } + if matched { + break; + } + } + if !matched { + count = count.saturating_add(1); + } + } + } + count +} + +/// Decode a binary netpbm grayscale image (`P5`), which is what `pdftoppm -gray` writes. +/// +/// Returns the dimensions and the pixel bytes. Only the maxval 255 form is accepted: a 16-bit +/// image would otherwise be read as twice as many 8-bit pixels and compared against nonsense. +pub fn decode_pgm(bytes: &[u8]) -> Option<(usize, usize, Vec)> { + let mut fields = Vec::new(); + let mut rest = bytes.strip_prefix(b"P5")?; + // Three whitespace-separated fields follow the magic: width, height, maxval. Comments run + // from `#` to end of line and may appear between any of them. + while fields.len() < 3 { + let skip = rest.iter().position(|b| !b.is_ascii_whitespace())?; + rest = rest.get(skip..)?; + if rest.first() == Some(&b'#') { + let end = rest.iter().position(|&b| b == b'\n')?; + rest = rest.get(end.checked_add(1)?..)?; + continue; + } + let end = rest.iter().position(u8::is_ascii_whitespace)?; + let field = rest.get(..end)?; + fields.push(std::str::from_utf8(field).ok()?.parse::().ok()?); + rest = rest.get(end..)?; + } + let (&width, &height, &maxval) = (fields.first()?, fields.get(1)?, fields.get(2)?); + if maxval != 255 { + return None; + } + // Exactly one whitespace byte separates the header from the data. + let pixels = rest.get(1..)?; + let wanted = width.checked_mul(height)?; + (pixels.len() >= wanted).then(|| (width, height, pixels.get(..wanted).unwrap_or_default().to_vec())) +} + +/// Decode the netpbm `P7` RGB_ALPHA image `pdfcraft-cli render --out x.pam` writes, flattening it +/// to grayscale the way a viewer would show it: composited over white, then luminance. +/// +/// The alpha is premultiplied, which is why the composite is `channel + white * (1 - alpha)` +/// rather than the usual `mix`. A page is normally opaque, but a transparent one must not come out +/// darker here than it looks on screen, or the comparison blames the renderer for the harness. +pub fn decode_pam_gray(bytes: &[u8]) -> Option<(usize, usize, Vec)> { + let header_end = find(bytes, b"ENDHDR\n")?.checked_add(b"ENDHDR\n".len())?; + let header = std::str::from_utf8(bytes.get(..header_end)?).ok()?; + if !header.starts_with("P7") { + return None; + } + let field = |name: &str| -> Option { header.lines().find_map(|l| l.strip_prefix(name)?.trim().parse::().ok()) }; + let (width, height) = (field("WIDTH")?, field("HEIGHT")?); + if field("DEPTH")? != 4 || field("MAXVAL")? != 255 || !header.contains("TUPLTYPE RGB_ALPHA") { + return None; + } + let data = bytes.get(header_end..)?; + let wanted = width.checked_mul(height)?.checked_mul(4)?; + if data.len() < wanted { + return None; + } + let mut out = Vec::with_capacity(width.checked_mul(height)?); + for px in data.get(..wanted)?.chunks_exact(4) { + let (&r, &g, &b, &a) = (px.first()?, px.get(1)?, px.get(2)?, px.get(3)?); + // Composite premultiplied RGB over white, in integer arithmetic that cannot overflow: + // each term is at most 255, and the sum is at most 255. + let over_white = |c: u8| u32::from(c).saturating_add(255u32.saturating_sub(u32::from(a))).min(255); + let (r, g, b) = (over_white(r), over_white(g), over_white(b)); + // Rec. 601 luma, the usual grayscale for this, rounded. Each term is at most 255*587, so + // the sum cannot overflow a u32. + let luma = (r * 299 + g * 587 + b * 114 + 500) / 1000; + out.push(u8::try_from(luma.min(255)).unwrap_or(255)); + } + Some((width, height, out)) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack.windows(needle.len()).position(|w| w == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn gray(w: usize, h: usize, fill: u8) -> Vec { + vec![fill; w * h] + } + + fn pam(w: usize, h: usize, px: &[u8]) -> Vec { + let mut out = format!("P7\nWIDTH {w}\nHEIGHT {h}\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n").into_bytes(); + out.extend_from_slice(px); + out + } + + #[test] + fn pam_flattens_to_the_grey_a_viewer_would_show() { + // Opaque white, opaque black, opaque mid-grey. + let px = [255, 255, 255, 255, 0, 0, 0, 255, 128, 128, 128, 255]; + let (w, h, g) = decode_pam_gray(&pam(3, 1, &px)).expect("a P7 image"); + assert_eq!((w, h), (3, 1)); + assert_eq!(g, vec![255, 0, 128]); + } + + /// A fully transparent pixel is white on screen, not black. Getting this wrong would make the + /// harness report a blank margin as a difference on every page that has one. + #[test] + fn transparent_pixels_composite_over_white() { + let px = [0, 0, 0, 0, 0, 0, 0, 128]; + let (_, _, g) = decode_pam_gray(&pam(2, 1, &px)).expect("a P7 image"); + assert_eq!(g.first().copied(), Some(255), "fully transparent is white"); + assert_eq!(g.get(1).copied(), Some(127), "half-transparent black over white is mid grey"); + } + + #[test] + fn hostile_pam_headers_are_rejected() { + let px = [0u8; 8]; + assert!(decode_pam_gray(&pam(2, 1, &px)).is_some()); + let cases: Vec> = vec![ + b"".to_vec(), + b"P7\n".to_vec(), + b"P7\nWIDTH 2\nHEIGHT 1\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\n".to_vec(), // no ENDHDR + pam(2, 1, &[0u8; 4]), // truncated data + b"P6\nWIDTH 2\nHEIGHT 1\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n".to_vec(), + b"P7\nWIDTH 2\nHEIGHT 1\nDEPTH 3\nMAXVAL 255\nTUPLTYPE RGB\nENDHDR\n".to_vec(), // no alpha + b"P7\nWIDTH 2\nHEIGHT 1\nDEPTH 4\nMAXVAL 65535\nTUPLTYPE RGB_ALPHA\nENDHDR\n".to_vec(), + b"P7\nWIDTH 99999999999999999999\nHEIGHT 1\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n".to_vec(), + ]; + for case in &cases { + assert!(decode_pam_gray(case).is_none(), "{:?}", String::from_utf8_lossy(case.get(..40).unwrap_or(case))); + } + // A declared area far larger than the data must fail on length, not allocate. + let huge = b"P7\nWIDTH 4000000000\nHEIGHT 4000000000\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n".to_vec(); + assert!(decode_pam_gray(&huge).is_none()); + } + + #[test] + fn identical_images_do_not_differ() { + let buf = gray(16, 16, 128); + let a = Gray::new(16, 16, &buf).expect("sized"); + let d = compare(a, a, 0).expect("same size"); + assert_eq!(d.differing, 0); + assert_eq!(d.max_delta, 0); + assert_eq!(d.fraction(), 0.0); + assert_eq!(d.total, 256); + } + + /// The whole point: a one-pixel shift of a stem is forgiven, because every moved pixel finds + /// its match next door. This is what makes the number mean something across renderers. + #[test] + fn a_one_pixel_shift_is_forgiven() { + let (w, h) = (9usize, 9usize); + let mut left = gray(w, h, 255); + let mut right = gray(w, h, 255); + for y in 0..h { + left[y * w + 4] = 0; // a vertical stem + right[y * w + 5] = 0; // the same stem, one pixel over + } + let a = Gray::new(w, h, &left).expect("sized"); + let b = Gray::new(w, h, &right).expect("sized"); + let d = compare(a, b, 8).expect("same size"); + assert_eq!(d.differing, 0, "a one-pixel shift must not count as a difference"); + // ...but it is still visible as a large single-pixel delta, which is the honest signal. + assert_eq!(d.max_delta, 255); + } + + /// And the point of the point: content that is simply absent is *not* forgiven, however + /// tolerant the neighbourhood test is. + #[test] + fn missing_content_is_not_forgiven() { + let (w, h) = (16usize, 16usize); + let blank = gray(w, h, 255); + let mut drawn = blank.clone(); + // A filled 6x6 block: its interior is further than one pixel from any white pixel. + for y in 5..11 { + for x in 5..11 { + drawn[y * w + x] = 0; + } + } + let a = Gray::new(w, h, &blank).expect("sized"); + let b = Gray::new(w, h, &drawn).expect("sized"); + let d = compare(a, b, 8).expect("same size"); + // Scanning the blank against the drawn image forgives the block's border, whose + // neighbourhood still reaches white, and counts only its 4x4 interior. Scanning the other + // way counts all 36 black pixels, since a blank image offers no dark match anywhere. The + // worse of the two is reported, so the whole block is -- which is the honest answer. + assert_eq!(d.differing, 36, "every pixel of a missing block must be counted"); + assert!(d.fraction() > 0.0); + } + + /// A three-pixel shift is beyond the 3x3 window and is reported. + #[test] + fn a_shift_wider_than_the_window_is_reported() { + let (w, h) = (16usize, 16usize); + let mut left = gray(w, h, 255); + let mut right = gray(w, h, 255); + for y in 0..h { + for x in 4..8 { + left[y * w + x] = 0; + } + for x in 11..15 { + right[y * w + x] = 0; + } + } + let a = Gray::new(w, h, &left).expect("sized"); + let b = Gray::new(w, h, &right).expect("sized"); + assert!(compare(a, b, 8).expect("same size").differing > 0); + } + + #[test] + fn tolerance_absorbs_small_differences_only() { + let flat = gray(8, 8, 100); + let near = gray(8, 8, 105); + let far = gray(8, 8, 200); + let a = Gray::new(8, 8, &flat).expect("sized"); + assert_eq!(compare(a, Gray::new(8, 8, &near).expect("sized"), 8).expect("ok").differing, 0); + assert_eq!(compare(a, Gray::new(8, 8, &far).expect("sized"), 8).expect("ok").differing, 64); + // Tolerance is inclusive at the boundary. + assert_eq!(compare(a, Gray::new(8, 8, &near).expect("sized"), 5).expect("ok").differing, 0); + assert_eq!(compare(a, Gray::new(8, 8, &near).expect("sized"), 4).expect("ok").differing, 64); + } + + /// Comparison is symmetric: neither argument order flatters one renderer. + #[test] + fn comparison_is_symmetric() { + let (w, h) = (12usize, 12usize); + let mut thin = gray(w, h, 255); + let mut thick = gray(w, h, 255); + for y in 0..h { + thin[y * w + 6] = 0; + for x in 4..9 { + thick[y * w + x] = 0; + } + } + let a = Gray::new(w, h, &thin).expect("sized"); + let b = Gray::new(w, h, &thick).expect("sized"); + let forward = compare(a, b, 8).expect("ok"); + let backward = compare(b, a, 8).expect("ok"); + assert_eq!(forward, backward, "argument order must not change the verdict"); + assert!(forward.differing > 0, "a 1px stem against a 5px bar is a real difference"); + } + + #[test] + fn different_sizes_are_an_error_not_a_score() { + let small = gray(4, 4, 0); + let large = gray(8, 8, 0); + let err = compare(Gray::new(4, 4, &small).expect("sized"), Gray::new(8, 8, &large).expect("sized"), 0).expect_err("mismatch"); + assert_eq!(err, SizeMismatch { a: (4, 4), b: (8, 8) }); + assert!(err.to_string().contains("4x4"), "{err}"); + } + + #[test] + fn a_buffer_that_is_not_the_stated_size_is_rejected() { + let buf = gray(4, 4, 0); + assert!(Gray::new(4, 4, &buf).is_some()); + assert!(Gray::new(5, 4, &buf).is_none()); + assert!(Gray::new(4, 5, &buf).is_none()); + assert!(Gray::new(usize::MAX, usize::MAX, &buf).is_none(), "the area must not overflow into a match"); + // A zero-sized image is representable and compares as no difference. + let empty: [u8; 0] = []; + let z = Gray::new(0, 0, &empty).expect("zero is a valid size"); + assert_eq!(compare(z, z, 0).expect("ok").fraction(), 0.0); + } + + #[test] + fn pgm_decodes_the_shape_pdftoppm_writes() { + let mut bytes = b"P5\n4 2\n255\n".to_vec(); + bytes.extend_from_slice(&[0, 1, 2, 3, 4, 5, 6, 7]); + let (w, h, px) = decode_pgm(&bytes).expect("a P5 image"); + assert_eq!((w, h), (4, 2)); + assert_eq!(px, vec![0, 1, 2, 3, 4, 5, 6, 7]); + } + + #[test] + fn pgm_handles_comments_and_odd_whitespace() { + let mut bytes = b"P5\n# written by pdftoppm\n 4\t2\n255 ".to_vec(); + bytes.extend_from_slice(&[9; 8]); + let (w, h, px) = decode_pgm(&bytes).expect("a commented P5 image"); + assert_eq!((w, h, px.len()), (4, 2, 8)); + } + + /// Malformed or hostile headers return `None` rather than panicking or inventing a size. + #[test] + fn hostile_pgm_headers_are_rejected() { + let cases: &[&[u8]] = &[ + b"", + b"P5", + b"P6\n4 2\n255\n", // colour, not grayscale + b"P5\n4 2\n65535\n", // 16-bit: would be read as twice the pixels + b"P5\n4 2\n255\n\x00", // truncated data + b"P5\n-4 2\n255\n", // negative + b"P5\n99999999999999999999 2\n255\n", + b"P5\n4\n", // header cut short + b"P5\n#unterminated comment", + ]; + for case in cases { + assert!(decode_pgm(case).is_none(), "{:?}", String::from_utf8_lossy(case)); + } + // A huge declared area must not allocate: it fails on the data length check. + let huge = b"P5\n4000000000 4000000000\n255\n".to_vec(); + assert!(decode_pgm(&huge).is_none()); + } +} diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index a612587d26..9f493e296f 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2025,10 +2025,10 @@ fn extract_still_prunes_separate_resources_when_many_pages_share_a_type3_font() assert_eq!(xobject_names(&part, 0), [b"ImA".to_vec()], "one shared font's direct resource graph is protected once"); } -// Stays ignored until CI installs Poppler: the pixel comparison has never run on a runner, so -// un-ignoring it here would turn an unverified test loose on everyone. See crates/oracle. +// Both sides of the comparison go through the same pdftoppm, so the assertion is about our two +// documents rather than about Poppler's output, and no Poppler version can change the verdict. +// It skips where pdftoppm is absent and fails where PDFCRAFT_ORACLES_REQUIRED is set. #[test] -#[ignore = "requires the external Poppler pdftoppm oracle"] fn extract_type3_shared_xobjects_matches_poppler_pixels() { let pdftoppm = pdfcraft_oracle::or_skip!(pdfcraft_oracle::pdftoppm()); let dir = std::env::temp_dir().join(format!("pdfcraft-type3-extract-{}", std::process::id())); diff --git a/docs/gaps.md b/docs/gaps.md index 4c235c1745..94bd360e7a 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -212,10 +212,25 @@ update the numbers. signature validation against Acrobat on synthetic fixtures; an interop suite that opens every kind of output we write (encrypted ones above all, after #774) in pdf.js, MuPDF and Poppler as oracle processes. -- **Evidence:** none exists; only text extraction has an oracle (word-F1 0.98 against `pdftotext`). - Acrobat captures must stay local in `plan/acrobat/` (clean-room, `AGENTS.md` §1.1). -- **Impact:** until it exists, every quality number here is a judgement. -- **Estimate:** counted in gap 2 (≈ 20–30 h of it). **Doc:** [target-app-parity.md](target-app-parity.md). +- **Progress (2026-10-14):** rendering now has an oracle too. `cargo xtask render-oracle` compares + page 1 of every corpus file against Poppler's `pdftoppm` and reports a disagreement rate: the + share of pixels with no near match in the other renderer's 3×3 neighbourhood, which forgives + antialiasing and subpixel placement but not missing or wrong content (`crates/oracle`, module + `pixels`). **First reading, 250 files at 72 dpi, tolerance 8: 229 compared, median 0.89%, mean + 2.16%, p90 5.35%, 25 within 0.1%.** It reports rather than gates — the corpus is not fetched on + CI, and a first reading of a borrowed renderer is a baseline to improve on, not a bar to clear. + The worst results cluster on JBIG2 `bitmap-*` files (`hayro-jbig2` carries 3 local patches) and + on ZapfDingbats, which points the renderer work at something specific for the first time. +- **Evidence:** text extraction (word-F1 0.98 against `pdftotext`) and now rendering (above); qpdf + checks written output on every CI run since `PDFCRAFT_ORACLES_REQUIRED`. Acrobat captures must + stay local in `plan/acrobat/` (clean-room, `AGENTS.md` §1.1). +- **Still missing:** Acrobat itself as the reference rather than other open renderers; form + calculation and signature validation; the interop suite that opens every kind of output we write + (encrypted ones above all, after #774) in pdf.js and MuPDF; and 15 of those 250 files could not + be compared at all because the two renderers round the page box to different pixel sizes. +- **Impact:** until it is complete, quality numbers outside rendering and text are judgements. +- **Estimate:** counted in gap 2 (≈ 20–30 h of it; roughly a third done). + **Doc:** [target-app-parity.md](target-app-parity.md). ### 13. Performance budgets and lazy loading of very large files diff --git a/xtask/src/gates.rs b/xtask/src/gates.rs index cf3d164ce8..fdc63ba808 100644 --- a/xtask/src/gates.rs +++ b/xtask/src/gates.rs @@ -239,15 +239,125 @@ pub fn text_oracle(args: &[String]) -> anyhow::Result<()> { Ok(()) } +/// Compare our rendering against Poppler's, page 1 of every PDF in a directory. +/// +/// This is the measured half of the fidelity gap (`docs/gaps.md` gap 12): until it exists, every +/// claim about rendering quality is a judgement. It reports rather than gates, because the corpus +/// is not fetched on CI and because a first reading of a borrowed renderer is a baseline to +/// improve on, not a bar to clear. +/// +/// The score is the share of pixels with no near match in the other renderer's 3x3 neighbourhood +/// (`pdfcraft_oracle::pixels`), which forgives antialiasing and subpixel placement but not +/// missing or wrong content. **Lower is better; it is a disagreement rate, not a quality score.** +pub fn render_oracle(args: &[String]) -> anyhow::Result<()> { + let limit: usize = flag_num(args, "--limit").unwrap_or(usize::MAX); + let dpi: u32 = flag_num(args, "--dpi").unwrap_or(72); + let tolerance: u8 = flag_num(args, "--tolerance").unwrap_or(8); + let dir = + args.iter().position(|a| a == "--dir").and_then(|i| args.get(i + 1)).map_or_else(|| root().join("corpus/pdfjs/test/pdfs"), PathBuf::from); + + let pdftoppm = match pdfcraft_oracle::pdftoppm() { + Ok(tool) => tool, + Err(absent) => bail!("{absent}"), + }; + run_args(&["build", "--release", "-p", "pdfcraft-cli"])?; + let cli = release_cli(); + + let mut files: Vec = std::fs::read_dir(&dir) + .with_context(|| format!("{}: no such directory (cargo xtask corpus, or pass --dir)", dir.display()))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|e| e.eq_ignore_ascii_case("pdf"))) + .collect(); + files.sort(); + + let out_pam = std::env::temp_dir().join(format!("pdfcraft-render-oracle-{}.pam", std::process::id())); + let (mut scores, mut sizes_differed, mut ours_failed, mut theirs_failed) = (Vec::new(), 0usize, 0usize, 0usize); + let mut worst: Vec<(f64, String)> = Vec::new(); + for file in files.iter().take(limit) { + let name = file.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + let ours = Command::new(&cli) + .args(["render", "--page", "1", "--dpi"]) + .arg(dpi.to_string()) + .arg("--out") + .arg(&out_pam) + .arg(file) + .status() + .ok() + .filter(std::process::ExitStatus::success) + .and_then(|_| std::fs::read(&out_pam).ok()) + .and_then(|b| pdfcraft_oracle::pixels::decode_pam_gray(&b)); + let theirs = + run_capture_bytes(pdftoppm.command().args(["-singlefile", "-gray", "-f", "1", "-l", "1", "-r"]).arg(dpi.to_string()).arg(file), 30) + .and_then(|b| pdfcraft_oracle::pixels::decode_pgm(&b)); + // An oracle that cannot read a file says nothing about our fidelity, and a file we cannot + // read is `xtask check`'s business, not this one's. Count each side and move on. + if ours.is_none() { + ours_failed = ours_failed.saturating_add(1); + } + if theirs.is_none() { + theirs_failed = theirs_failed.saturating_add(1); + } + let (Some((aw, ah, a)), Some((bw, bh, b))) = (ours, theirs) else { continue }; + let (Some(a), Some(b)) = (pixels_of(aw, ah, &a), pixels_of(bw, bh, &b)) else { continue }; + match pdfcraft_oracle::pixels::compare(a, b, tolerance) { + Ok(diff) => { + scores.push(diff.percent()); + worst.push((diff.percent(), name)); + } + // Different rounding of the page box at this scale: not a fidelity signal. + Err(_) => sizes_differed = sizes_differed.saturating_add(1), + } + } + let _ = std::fs::remove_file(&out_pam); + + if scores.is_empty() { + bail!( + "no comparable files in {} ({sizes_differed} size mismatches, {ours_failed} we could not render, {theirs_failed} Poppler could not)", + dir.display() + ); + } + scores.sort_by(f64::total_cmp); + worst.sort_by(|a, b| b.0.total_cmp(&a.0)); + let median = scores.get(scores.len() / 2).copied().unwrap_or(0.0); + let mean = scores.iter().sum::() / scores.len() as f64; + let p90 = scores.get(scores.len().saturating_mul(9) / 10).copied().unwrap_or(0.0); + let clean = scores.iter().filter(|s| **s < 0.1).count(); + println!("render oracle vs Poppler at {dpi} dpi, tolerance {tolerance} (disagreement rate, lower is better)"); + println!(" {} compared: median {median:.2}%, mean {mean:.2}%, p90 {p90:.2}%", scores.len()); + println!(" {clean} within 0.1%; {sizes_differed} size mismatches, {ours_failed} we could not render, {theirs_failed} Poppler could not"); + println!("worst:"); + for (score, name) in worst.iter().take(12) { + println!(" {score:6.2}% {name}"); + } + Ok(()) +} + +/// The value after `name` on the command line, parsed, or `None` when it is absent or malformed. +fn flag_num(args: &[String], name: &str) -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i.checked_add(1)?)).and_then(|v| v.parse().ok()) +} + +fn pixels_of(w: usize, h: usize, buf: &[u8]) -> Option> { + pdfcraft_oracle::pixels::Gray::new(w, h, buf) +} + fn run_capture(cmd: &mut Command, timeout_s: u64) -> Option { + // `read_to_string` left the buffer empty when the output was not UTF-8; `from_utf8` keeps + // exactly that behaviour now that the capture itself is byte-oriented. + Some(String::from_utf8(run_capture_bytes(cmd, timeout_s)?).unwrap_or_default()) +} + +/// [`run_capture`] for output that is not text — an oracle's image on stdout, for instance. +fn run_capture_bytes(cmd: &mut Command, timeout_s: u64) -> Option> { let mut child = cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::null()).spawn().ok()?; let start = std::time::Instant::now(); - let mut out = String::new(); + let mut out = Vec::new(); let mut stdout = child.stdout.take()?; let reader = std::thread::spawn(move || { use std::io::Read; - let mut s = String::new(); - let _ = stdout.read_to_string(&mut s); + let mut s = Vec::new(); + let _ = stdout.read_to_end(&mut s); s }); loop { @@ -262,7 +372,7 @@ fn run_capture(cmd: &mut Command, timeout_s: u64) -> Option { Err(_) => return None, } } - out.push_str(&reader.join().ok()?); + out.extend_from_slice(&reader.join().ok()?); Some(out) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 70741d22b7..2e91afe2ac 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -33,6 +33,11 @@ const COMMANDS: &[(&str, &str, Command)] = &[ macos_integration::run, ), ("text-oracle", "Compare text extraction with pdftotext over corpus/ (word F1; target median ≥ 0.97)", gates::text_oracle), + ( + "render-oracle", + "Compare page rendering with Poppler over corpus/ (pixel disagreement rate; --dir, --dpi, --limit, --tolerance)", + gates::render_oracle, + ), ("screenshots", "Regenerate the README screenshots in docs/images/ and their ATTRIBUTION entries", screenshots::run), ("models", "Fetch the OCR models (ATTRIBUTION.toml kind = \"model\") into assets/models/, verified by SHA-256", assets::models), ("demo-pdf", "Build dist/demo/pdfcraft-showcase.pdf (needs Google Chrome or Chromium)", demo_pdf::run),