From 749636fb2ec82518b4de2eb985cc419212eae4cf Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 23:04:23 -0400 Subject: [PATCH] LR-IMP-FORMATS: decode table-less Nikon 34713 NEFs stored as packed 12/14-bit data --- crates/engine/src/media.rs | 2 +- crates/raw/src/vendor/nef.rs | 223 ++++++++++++++++++++++++++++++++++- 2 files changed, 218 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 22d76528d..e6c438ad7 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 64; +pub const RENDER_CACHE_VERSION: u64 = 65; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/vendor/nef.rs b/crates/raw/src/vendor/nef.rs index 5076c6df9..5fbf83414 100644 --- a/crates/raw/src/vendor/nef.rs +++ b/crates/raw/src/vendor/nef.rs @@ -7,6 +7,9 @@ //! rule documented there; files it can't decode are reported as [`RawError::Unsupported`] and their embedded previews still work. So are High Efficiency NEFs //! (HE / HE★, Z 8, Z 9, Z 6III, Z f): they keep compression 34713 but carry a wavelet codestream that starts with //! the JPEG XS markers SOC + CAP (ISO/IEC 21122-1, `FF10 FF50`) and have no `0x0096` table (issue #193). +//! Strips labelled 34713 without a `0x0096` +//! linearization table are packed uncompressed data in several measured layouts ([`PackedLayout`]); Nikon small raw +//! (sRAW) and the D1X's genuinely Huffman-coded strip stay refused. use super::{nefc, white_from_data}; use crate::tiffraw::{Packing, check_image, read_image}; @@ -342,8 +345,83 @@ fn uncompressed(bytes: &[u8], info: &ImageInfo, order: ByteOrder, mn: Option<&ma read_image(bytes, &top, order, packing) } -/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`. Decodes the first -/// `rows` rows of the `info.height`. +/// Packed layouts of strips that are labelled compression 34713 but have no linearization table (maker note +/// `0x0096`): they hold plain packed samples, not Huffman codes. The layouts were measured from CC0 raw.pixls.us +/// files by a clean-room black-box analysis (strip byte-count identity, same-colour smoothness and correlation with +/// the camera JPEG against at least eight alternative packings). Every layout is a single strip with +/// `stride = StripByteCounts / height`, selected only when `stride * height` equals the strip size exactly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PackedLayout { + /// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned. + /// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes + /// padded to 16 bytes. + Lsb { stride: usize }, + /// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The + /// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image. + Chunked16 { stride: usize }, +} + +fn packed_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + if w == 0 || h == 0 || !matches!(bits, 12 | 14) { + return None; + } + let strip = usize::try_from(strip).ok()?; + if !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let tight = w.checked_mul(bits as usize)?.div_ceil(8); + let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None }; + let padded = tight.checked_next_multiple_of(16)?; + let lsb_ok = match bits { + 12 => stride == tight || stride == padded, + _ => stride == padded, + }; + if lsb_ok { + return Some(PackedLayout::Lsb { stride }); + } + None +} + +fn d100_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + let strip = usize::try_from(strip).ok()?; + if bits != 12 || h == 0 || !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let per_row = stride / 16 * 10; + (stride.is_multiple_of(16) && per_row >= w && per_row < w.checked_add(10)?).then_some(PackedLayout::Chunked16 { stride }) +} + +/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero. +fn unpack_packed(src: &[u8], w: usize, h: usize, bits: u32, layout: PackedLayout) -> Vec { + use rayon::prelude::*; + let mut out = vec![0u16; w * h]; + let stride = match layout { + PackedLayout::Lsb { stride } | PackedLayout::Chunked16 { stride } => stride, + }; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let Some(line) = src.get(y * stride..) else { return }; + let line = &line[..stride.min(line.len())]; + match layout { + PackedLayout::Lsb { .. } => crate::unpack::unpack_lsb(line, bits, row), + PackedLayout::Chunked16 { .. } => { + let mut samples = vec![0u16; line.len() / 16 * 10]; + for (i, dst) in samples.chunks_mut(10).enumerate() { + if let Some(chunk) = line.get(i * 16..i * 16 + 15) { + crate::unpack::unpack_msb(chunk, 12, dst); + } + } + let n = row.len().min(samples.len()); + row[..n].copy_from_slice(&samples[..n]); + } + } + }); + out +} + +/// Nikon Huffman-compressed strip (see [`super::nefc`]); the decode table is maker note `0x0096`. Without a table the +/// strip may be packed uncompressed data (see [`PackedLayout`]). fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, rows: usize) -> Result { let starts_with = |magic: &[u8]| { let start = info.chunks(bytes.len() as u64).first().and_then(|c| usize::try_from(c.offset).ok()); @@ -352,10 +430,9 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, if starts_with(&JPEG_XS_START) { return Err(RawError::Unsupported(HIGH_EFFICIENCY.into())); } - // (some Z bodies label uncompressed, row-padded data 34713 without a table: not handled here yet) - let table = mn - .and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order))) - .ok_or_else(|| RawError::Unsupported("Nikon compressed NEF without a linearization table (maker note 0x96)".into()))?; + let Some(table) = mn.and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order))) else { + return packed(bytes, info, rows); + }; if info.samples_per_pixel != 1 { return Err(RawError::Unsupported(format!("Nikon compressed NEF with {} samples per pixel", info.samples_per_pixel))); } @@ -370,6 +447,32 @@ fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, Ok(RawData::U16(nefc::decode_rows(src, info.width as usize, info.height as usize, rows, bits, &table)?)) } +/// A table-less "compressed" strip: packed samples when the strip size matches a measured layout exactly. +fn packed(bytes: &[u8], info: &ImageInfo, rows: usize) -> Result { + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let chunks = info.chunks(bytes.len() as u64); + let layout = match chunks.as_slice() { + [c] if info.samples_per_pixel == 1 => packed_layout(w, h, bits, c.len).or_else(|| d100_layout(w, h, bits, c.len)), + _ => None, + }; + let (Some(layout), [c]) = (layout, chunks.as_slice()) else { + let small = bits == 12 && chunks.len() == 1 && w.checked_mul(h).and_then(|n| n.checked_mul(3)).is_some_and(|n| u64::try_from(n).ok() == Some(chunks[0].len)); + return Err(RawError::Unsupported(if small { + "Nikon small raw (sRAW) not decoded yet".into() + } else { + "Nikon compressed NEF without a linearization table (maker note 0x96)".into() + })); + }; + let end = c.offset.saturating_add(c.len).min(bytes.len() as u64); + let src = usize::try_from(c.offset).ok().zip(usize::try_from(end).ok()).and_then(|(a, b)| bytes.get(a..b)); + let src = src.ok_or_else(|| RawError::Corrupt("NEF: image data outside the file".into()))?; + if (w as u64).saturating_mul(h as u64) > crate::MAX_SAMPLES as u64 { + return Err(RawError::Limit("image too large")); + } + Ok(RawData::U16(unpack_packed(src, w, rows.min(h), bits, layout))) +} + #[cfg(test)] mod tests { use super::*; @@ -819,4 +922,112 @@ mod tests { let short = nef(34713, 14, vec![vec![0xff, 0x10]], 1, 1, 1); assert!(matches!(crate::decode(&short), Err(RawError::Unsupported(w)) if w.contains("linearization table"))); } + + fn samples(n: usize, bits: u32) -> Vec { + (0..n).map(|i| (((i * 2654435761usize) >> 7) as u32 % (1 << bits)) as u16).collect() + } + + fn pack_lsb(row: &[u16], bits: u32) -> Vec { + let (mut out, mut acc, mut n) = (Vec::new(), 0u64, 0u32); + for &v in row { + acc |= (v as u64) << n; + n += bits; + while n >= 8 { + out.push(acc as u8); + acc >>= 8; + n -= 8; + } + } + if n > 0 { + out.push(acc as u8); + } + out + } + + fn pack_msb12(row: &[u16]) -> Vec { + let mut out = Vec::new(); + for pair in row.chunks(2) { + let (a, b) = (pair[0], pair.get(1).copied().unwrap_or(0)); + out.extend([(a >> 4) as u8, ((a & 15) << 4) as u8 | (b >> 8) as u8, b as u8]); + } + out + } + + /// Rows of `samples` packed by `row`, each padded with `pad` bytes of 0xA5 (never decoded). + fn strip(data: &[u16], w: usize, pad: usize, row: impl Fn(&[u16]) -> Vec) -> Vec { + data.chunks(w).flat_map(|r| row(r).into_iter().chain(std::iter::repeat_n(0xA5, pad))).collect() + } + + fn decoded(strip: Vec, bits: u16, w: u32, h: u32) -> Result> { + let bytes = nef(34713, bits, vec![strip], w, h, h); + match crate::decode(&bytes)?.data { + RawData::U16(d) => Ok(d), + _ => Err(RawError::Unsupported("float".into())), + } + } + + #[test] + fn packed_34713_layouts_round_trip() { + let (w, h) = (40usize, 6usize); + let d12 = samples(w * h, 12); + let d14 = samples(w * h, 14); + // group A: 12-bit, 3w/2 bytes per row + assert_eq!(decoded(strip(&d12, w, 0, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group B: 12-bit, rows padded to 16 bytes (60 -> 64) + assert_eq!(decoded(strip(&d12, w, 4, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group C: 14-bit, 70 bytes -> 80 + assert_eq!(decoded(strip(&d14, w, 10, |r| pack_lsb(r, 14)), 14, 40, 6).unwrap(), d14); + // group E: 16-byte chunks, 15 data bytes (ten samples) + a zero byte; 16 chunks = 160 samples (at 40 wide the + // 64-byte stride would be group B's) + let (w, h) = (160usize, 3usize); + let d12 = samples(w * h, 12); + let e = strip(&d12, w, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + assert_eq!(e.len(), 256 * h); + assert_eq!(decoded(e, 12, 160, 3).unwrap(), d12); + } + + #[test] + fn packed_d100_rows_carry_extra_columns() { + // IFD width 154, rows hold 160 samples (16 chunks); the last 6 are dropped + let (w, wide, h) = (154usize, 160usize, 3usize); + let wide_data = samples(wide * h, 12); + let e = strip(&wide_data, wide, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + let got = decoded(e, 12, w as u32, h as u32).unwrap(); + let want: Vec = wide_data.chunks(wide).flat_map(|r| r[..w].iter().copied()).collect(); + assert_eq!(got, want); + } + + #[test] + fn packed_34713_wrong_sizes_stay_refused() { + let d12 = samples(40 * 6, 12); + let good = strip(&d12, 40, 0, |r| pack_lsb(r, 12)); + // one byte more or fewer, a stride that is neither tight nor 16-aligned, a 13-bit file + for len in [good.len() - 1, good.len() + 1, good.len() + 6] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_)))); + // 14-bit rows must be padded to 16 bytes + let d14 = samples(40 * 6, 14); + assert!(matches!(decoded(strip(&d14, 40, 0, |r| pack_lsb(r, 14)), 14, 40, 6), Err(RawError::Unsupported(_)))); + // 3 bytes per IFD-width pixel is a Nikon small raw: still refused, with its own reason + let Err(RawError::Unsupported(why)) = decoded(vec![0; 3 * 40 * 6], 12, 40, 6) else { panic!("small raw decoded") }; + assert!(why.contains("sRAW"), "{why}"); + } + + #[test] + fn packed_34713_short_input_does_not_panic() { + let (w, h) = (40usize, 6usize); + for layout in [PackedLayout::Lsb { stride: 64 }, PackedLayout::Lsb { stride: 80 }, PackedLayout::Chunked16 { stride: 64 }] { + for len in [0, 1, 15, 16, 17, 63, 64, 65, 200] { + let bits = if matches!(layout, PackedLayout::Lsb { stride: 80 }) { 14 } else { 12 }; + assert_eq!(unpack_packed(&vec![0xFF; len], w, h, bits, layout).len(), w * h); + } + } + // a strip that claims more bytes than the file has + let mut bytes = nef(34713, 12, vec![vec![0x55; 60 * 6]], 40, 6, 6); + bytes.truncate(bytes.len() - 100); + let _ = crate::decode(&bytes); + } }