mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 00:52:38 +00:00
fix: decompress single-file zst/bz2/xz/gz disk images (#1048)
* fix: decompress single-file zst/bz2/xz/gz disk images into tmpdir ExtractDiskImageFromFile only handled tar archives and a broken plain .xz branch that ran xz -dk beside the source and then tried to parse tar chatter for a path, so it silently returned an empty string. Talos (.raw.zst, .raw.xz), Fedora CoreOS (.qcow2.xz, .raw.xz) and Flatcar (.img.bz2) publish only single-file compressed artifacts, so none of them could be imported. Add streaming decompression for .zst, .bz2, .xz and .gz that writes into the caller's tmpdir and returns that path directly, instead of inferring it from tar output. A decompressed .qcow2 payload now flows through the existing qcow2-to-raw qemu-img conversion via a shared finalizeDiskImage helper. zstd uses klauspost/compress, already a transitive dependency via containerd, for a pure-Go streaming decoder. bz2 and gzip use the standard library. xz shells out to the xz binary since no pure-Go xz decoder is already a dependency, and the existing .tar.xz branch already requires that binary on the host via tar's -J flag, so this adds no new host requirement. * fix: write sparse holes when decompressing single-file disk images Verified live on env13: a Talos raw.zst imported successfully but the decompressed output was written literally rather than sparsely, an 11 GiB file on disk for content that decompresses to 183 MiB with xz -dc | dd conv=sparse. This is a regression relative to the qcow2-to-raw path beside it, which already writes sparse via qemu-img convert. decompressToTemp now reads in 1 MiB blocks, seeks past all-zero blocks instead of writing them, and truncates to the final size so a trailing zero run is still recorded as a hole. 1 MiB amortises the read/seek syscall overhead over a multi-GiB image while still being small enough to catch the large contiguous zero runs that dominate a sparse raw disk; a zero run shorter than one block is written literally, which is the safe direction for a false negative to fail in. zstd, bzip2 and gzip all route through decompressToTemp, so all three gain sparse output from this change. The plain-.xz path shells out to the xz binary with output piped straight to a file and does not go through decompressToTemp, so it is unaffected by this fix and remains a known follow-up. * fix: make plain-.xz decompression sparse too decompressXz shelled its output straight to a file, bypassing the sparse-hole handling just added to decompressToTemp for zstd, bzip2 and gzip. .raw.xz is the format Talos and Flatcar actually publish alongside .zst, so this path is the one most likely hit in practice, and left dense the exact disk-size regression this fix targets still fires for it. exec.Cmd already runs its own copier goroutine when Stdout is not an *os.File, and Wait joins it. decompressXz now hands the xz process an io.Pipe write end and reuses decompressToTemp on the read end, so plain .xz gets identical sparse-hole handling to the other codecs. A separate goroutine closes the pipe with the process's exit error once Wait returns, so a read past the end of a failed decompression surfaces that error, stderr included, instead of a bare EOF; the error path also closes the read end explicitly so a stuck copier cannot wedge Wait forever if decompressToTemp gives up early.
This commit is contained in:
@@ -19,6 +19,7 @@ require (
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/klauspost/cpuid/v2 v2.4.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/minio/highwayhash v1.0.4
|
||||
@@ -135,7 +136,6 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/josharian/native v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.19.2 // indirect
|
||||
github.com/klauspost/reedsolomon v1.14.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
|
||||
+235
-60
@@ -3,6 +3,8 @@ package utils
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
@@ -27,6 +29,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
var ErrQCOWDetected = errors.New("qcow format detected")
|
||||
@@ -594,6 +597,11 @@ var AvailableImages = map[string]Images{
|
||||
},
|
||||
}
|
||||
|
||||
// ExtractDiskImageFromFile turns a downloaded artifact at imagepath into a bootable disk
|
||||
// image and returns its path. tar-based archives are expanded into tmpdir and their listing
|
||||
// is scanned for the member that carries the payload; single-file compressed artifacts
|
||||
// (.zst, .bz2, .xz, .gz) are streamed straight into tmpdir under their own decompressed name.
|
||||
// The source artifact at imagepath is never modified or overwritten.
|
||||
func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string, err error) {
|
||||
var args []string
|
||||
var execCmd string
|
||||
@@ -605,77 +613,56 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
|
||||
|
||||
imagefile := filepath.Base(imagepath)
|
||||
|
||||
if strings.HasSuffix(imagefile, ".raw") || strings.HasSuffix(imagefile, ".img") || strings.HasSuffix(imagefile, ".qcow2") || strings.HasSuffix(imagefile, ".qcow") {
|
||||
switch {
|
||||
case strings.HasSuffix(imagefile, ".raw") || strings.HasSuffix(imagefile, ".img") || strings.HasSuffix(imagefile, ".qcow2") || strings.HasSuffix(imagefile, ".qcow"):
|
||||
path, err := filepath.Abs(imagepath)
|
||||
if err != nil {
|
||||
return path, err
|
||||
}
|
||||
err = validateDiskImagePath(path)
|
||||
if errors.Is(err, ErrQCOWDetected) {
|
||||
extractpath := fmt.Sprintf("%s/%s", tmpdir, imagefile)
|
||||
extractpath = strings.TrimSuffix(extractpath, ".qcow2") + ".raw"
|
||||
|
||||
args = []string{
|
||||
"convert",
|
||||
"-f",
|
||||
"qcow2",
|
||||
"-O",
|
||||
"raw",
|
||||
imagepath,
|
||||
"-C",
|
||||
extractpath,
|
||||
}
|
||||
|
||||
execCmd = "qemu-img"
|
||||
|
||||
cmd := exec.Command(execCmd, args...)
|
||||
_, err = cmd.Output()
|
||||
|
||||
if err != nil {
|
||||
return path, err
|
||||
}
|
||||
|
||||
return extractpath, nil
|
||||
}
|
||||
|
||||
return path, err
|
||||
} else if strings.HasSuffix(imagefile, ".tar.xz") {
|
||||
args = []string{
|
||||
"xfvJ",
|
||||
imagepath,
|
||||
"-C",
|
||||
tmpdir,
|
||||
}
|
||||
return finalizeDiskImage(path, tmpdir)
|
||||
|
||||
case strings.HasSuffix(imagefile, ".tar.xz"):
|
||||
args = []string{"xfvJ", imagepath, "-C", tmpdir}
|
||||
execCmd = "tar"
|
||||
} else if strings.HasSuffix(imagefile, ".tar.gz") || strings.HasSuffix(imagefile, ".tgz") {
|
||||
args = []string{
|
||||
"xfvz",
|
||||
imagepath,
|
||||
"-C",
|
||||
tmpdir,
|
||||
}
|
||||
|
||||
case strings.HasSuffix(imagefile, ".tar.gz") || strings.HasSuffix(imagefile, ".tgz"):
|
||||
args = []string{"xfvz", imagepath, "-C", tmpdir}
|
||||
execCmd = "tar"
|
||||
} else if strings.HasSuffix(imagefile, ".tar") {
|
||||
args = []string{
|
||||
"xfv",
|
||||
imagepath,
|
||||
"-C",
|
||||
tmpdir,
|
||||
}
|
||||
|
||||
case strings.HasSuffix(imagefile, ".tar"):
|
||||
args = []string{"xfv", imagepath, "-C", tmpdir}
|
||||
execCmd = "tar"
|
||||
} else if strings.HasSuffix(imagefile, ".xz") {
|
||||
args = []string{
|
||||
"-dk",
|
||||
imagepath,
|
||||
}
|
||||
|
||||
execCmd = "xz"
|
||||
} else {
|
||||
err = errors.New("unsupported filetype")
|
||||
return diskimage, err
|
||||
case strings.HasSuffix(imagefile, ".zst"):
|
||||
extracted, err := decompressZstd(imagepath, tmpdir)
|
||||
if err != nil {
|
||||
return diskimage, err
|
||||
}
|
||||
return finalizeDiskImage(extracted, tmpdir)
|
||||
|
||||
case strings.HasSuffix(imagefile, ".bz2"):
|
||||
extracted, err := decompressBzip2(imagepath, tmpdir)
|
||||
if err != nil {
|
||||
return diskimage, err
|
||||
}
|
||||
return finalizeDiskImage(extracted, tmpdir)
|
||||
|
||||
case strings.HasSuffix(imagefile, ".xz"):
|
||||
extracted, err := decompressXz(imagepath, tmpdir)
|
||||
if err != nil {
|
||||
return diskimage, err
|
||||
}
|
||||
return finalizeDiskImage(extracted, tmpdir)
|
||||
|
||||
case strings.HasSuffix(imagefile, ".gz"):
|
||||
extracted, err := decompressGzip(imagepath, tmpdir)
|
||||
if err != nil {
|
||||
return diskimage, err
|
||||
}
|
||||
return finalizeDiskImage(extracted, tmpdir)
|
||||
|
||||
default:
|
||||
return diskimage, errors.New("unsupported filetype")
|
||||
}
|
||||
|
||||
cmd := exec.Command(execCmd, args...)
|
||||
@@ -690,6 +677,194 @@ func ExtractDiskImageFromFile(imagepath string, tmpdir string) (diskimage string
|
||||
return diskimage, err
|
||||
}
|
||||
|
||||
// finalizeDiskImage runs the file(1) sniff gate on path and, if it detects a QCOW2 payload,
|
||||
// converts it to raw inside tmpdir. path may be the original source (already .raw/.img/.qcow2)
|
||||
// or a file this package just decompressed into tmpdir; either way the source is left intact
|
||||
// and only the raw output lands in tmpdir.
|
||||
func finalizeDiskImage(path string, tmpdir string) (diskimage string, err error) {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return absPath, err
|
||||
}
|
||||
|
||||
err = validateDiskImagePath(absPath)
|
||||
if errors.Is(err, ErrQCOWDetected) {
|
||||
base := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(absPath), ".qcow2"), ".qcow")
|
||||
extractpath := filepath.Join(tmpdir, base+".raw")
|
||||
|
||||
cmd := exec.Command("qemu-img", "convert", "-f", "qcow2", "-O", "raw", absPath, "-C", extractpath)
|
||||
if _, err := cmd.Output(); err != nil {
|
||||
return absPath, err
|
||||
}
|
||||
|
||||
return extractpath, nil
|
||||
}
|
||||
|
||||
return absPath, err
|
||||
}
|
||||
|
||||
// sparseBlockSize is the unit decompressToTemp checks for an all-zero run before punching a
|
||||
// hole instead of writing it. 1 MiB comfortably amortises the read/seek syscall overhead over
|
||||
// a multi-GiB image and still catches the large contiguous zero runs that dominate a sparse
|
||||
// raw disk (unused extents span tens of MiB or more); a zero run shorter than one block simply
|
||||
// gets written literally, which is the safe direction for a false negative to fail in.
|
||||
const sparseBlockSize = 1 << 20
|
||||
|
||||
// decompressToTemp streams r into a new sparse file inside tmpdir, named after src with
|
||||
// suffix stripped, and returns that path. All-zero blocks are skipped via Seek rather than
|
||||
// written, so a mostly-unused multi-GiB raw image (e.g. a Talos artifact, ~11 GiB apparent but
|
||||
// only a few hundred MiB of real content) costs disk space proportional to its real content —
|
||||
// matching the existing qcow2-to-raw path, which already writes sparse via qemu-img convert.
|
||||
// It never writes into src's own directory, so a multi-GiB source is never duplicated beside
|
||||
// itself while the caller-provided tmpdir receives the copy.
|
||||
func decompressToTemp(src, tmpdir, suffix string, r io.Reader) (string, error) {
|
||||
outPath := filepath.Join(tmpdir, strings.TrimSuffix(filepath.Base(src), suffix))
|
||||
|
||||
out, err := os.Create(outPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
var size int64
|
||||
buf := make([]byte, sparseBlockSize)
|
||||
for {
|
||||
n, readErr := io.ReadFull(r, buf)
|
||||
if n > 0 {
|
||||
block := buf[:n]
|
||||
if isZeroBlock(block) {
|
||||
if _, err := out.Seek(int64(n), io.SeekCurrent); err != nil {
|
||||
return "", fmt.Errorf("decompress %s: seek hole: %w", src, err)
|
||||
}
|
||||
} else if _, err := out.Write(block); err != nil {
|
||||
return "", fmt.Errorf("decompress %s: %w", src, err)
|
||||
}
|
||||
size += int64(n)
|
||||
}
|
||||
if readErr != nil {
|
||||
if errors.Is(readErr, io.EOF) || errors.Is(readErr, io.ErrUnexpectedEOF) {
|
||||
break
|
||||
}
|
||||
return "", fmt.Errorf("decompress %s: %w", src, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
// A trailing all-zero block only advances the file position via Seek, which does not
|
||||
// extend the file on its own; Truncate records that final hole at the right size.
|
||||
if err := out.Truncate(size); err != nil {
|
||||
return "", fmt.Errorf("decompress %s: truncate: %w", src, err)
|
||||
}
|
||||
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
// isZeroBlock reports whether every byte in b is zero.
|
||||
func isZeroBlock(b []byte) bool {
|
||||
for _, c := range b {
|
||||
if c != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// decompressZstd streams a single-file .zst artifact (e.g. Talos raw.zst) into tmpdir using
|
||||
// klauspost/compress, already a transitive dependency via containerd, so no new module is
|
||||
// added. It is a pure-Go streaming decoder, unlike shelling out to a zstd binary that may not
|
||||
// exist on a customer node.
|
||||
func decompressZstd(src, tmpdir string) (string, error) {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
zr, err := zstd.NewReader(f)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("zstd reader for %s: %w", src, err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
return decompressToTemp(src, tmpdir, ".zst", zr)
|
||||
}
|
||||
|
||||
// decompressBzip2 streams a single-file .bz2 artifact (e.g. Flatcar img.bz2) into tmpdir
|
||||
// using the standard library's decompress-only reader; no bzip2 encoder exists in this
|
||||
// codebase, so there is nothing to conflate this with a compression path.
|
||||
func decompressBzip2(src, tmpdir string) (string, error) {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return decompressToTemp(src, tmpdir, ".bz2", bzip2.NewReader(f))
|
||||
}
|
||||
|
||||
// decompressGzip streams a single-file .gz artifact into tmpdir using the standard library.
|
||||
func decompressGzip(src, tmpdir string) (string, error) {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
gr, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("gzip reader for %s: %w", src, err)
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
return decompressToTemp(src, tmpdir, ".gz", gr)
|
||||
}
|
||||
|
||||
// decompressXz shells out to the xz binary, piping its decompressed stdout through
|
||||
// decompressToTemp via an io.Pipe so a plain .xz image gets the same sparse-hole handling as
|
||||
// the other codecs rather than a dense copy. No pure-Go xz decoder is already a dependency of
|
||||
// this module, and the existing .tar.xz branch already requires the xz binary on the host
|
||||
// (tar's -J flag invokes it internally), so this adds no new host requirement — only makes the
|
||||
// existing one explicit for the plain-.xz case.
|
||||
func decompressXz(src, tmpdir string) (string, error) {
|
||||
var stderr bytes.Buffer
|
||||
cmd := exec.Command("xz", "-dc", src)
|
||||
pr, pw := io.Pipe()
|
||||
cmd.Stdout = pw
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", fmt.Errorf("start xz for %s: %w", src, err)
|
||||
}
|
||||
|
||||
// cmd.Stdout is not an *os.File, so exec.Cmd already runs its own copier goroutine from
|
||||
// the child's real pipe into pw; Wait joins it. This goroutine closes pw with the
|
||||
// process's exit error once Wait returns, so a Read past the end of the stream on pr
|
||||
// surfaces that error instead of a bare EOF.
|
||||
waitErrCh := make(chan error, 1)
|
||||
go func() {
|
||||
waitErr := cmd.Wait()
|
||||
waitErrCh <- waitErr
|
||||
pw.CloseWithError(waitErr)
|
||||
}()
|
||||
|
||||
outPath, err := decompressToTemp(src, tmpdir, ".xz", pr)
|
||||
if err != nil {
|
||||
// decompressToTemp gave up before draining the pipe (e.g. a local write failure), so
|
||||
// close the read end now — otherwise the copier goroutine above would stay blocked
|
||||
// writing into it forever, wedging cmd.Wait() and leaking both goroutines.
|
||||
pr.CloseWithError(err)
|
||||
}
|
||||
waitErr := <-waitErrCh
|
||||
|
||||
if waitErr != nil {
|
||||
return "", fmt.Errorf("xz decompress %s: %w: %s", src, waitErr, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("decompress %s: %w", src, err)
|
||||
}
|
||||
|
||||
return outPath, nil
|
||||
}
|
||||
|
||||
func extractDiskImagePath(imagedir string, output []byte) (diskimage string, err error) {
|
||||
reader := bytes.NewReader(output)
|
||||
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// resolveServingAMI filters purely on these two tags and never on the image
|
||||
@@ -131,3 +140,354 @@ func TestSelectNewestImage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// rawFixture returns the bytes of the repo's known-good MBR fixture, so a compressed copy of
|
||||
// it decodes to something validateDiskImagePath's file(1) sniff already accepts elsewhere.
|
||||
func rawFixture(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(filepath.Join("..", "..", "tests", "unit-test-disk-image.raw"))
|
||||
if err != nil {
|
||||
t.Fatalf("read raw fixture: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// qcow2FixtureBytes converts the raw fixture into a real QCOW2 image via qemu-img, in a
|
||||
// scratch dir that is discarded once built, so the .qcow2.xz case below exercises genuine
|
||||
// QCOW2 framing rather than a hand-rolled header.
|
||||
func qcow2FixtureBytes(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("qemu-img"); err != nil {
|
||||
t.Skip("qemu-img not found, skipping qcow2 conversion case")
|
||||
}
|
||||
|
||||
scratch := t.TempDir()
|
||||
rawPath := filepath.Join(scratch, "src.raw")
|
||||
if err := os.WriteFile(rawPath, rawFixture(t), 0644); err != nil {
|
||||
t.Fatalf("write raw source: %v", err)
|
||||
}
|
||||
|
||||
qcowPath := filepath.Join(scratch, "src.qcow2")
|
||||
cmd := exec.Command("qemu-img", "convert", "-f", "raw", "-O", "qcow2", rawPath, qcowPath)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("qemu-img convert to qcow2: %v: %s", err, out)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(qcowPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read qcow2 fixture: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func gzipBytes(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
if _, err := gw.Write(data); err != nil {
|
||||
t.Fatalf("gzip write: %v", err)
|
||||
}
|
||||
if err := gw.Close(); err != nil {
|
||||
t.Fatalf("gzip close: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func zstdBytes(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw, err := zstd.NewWriter(&buf)
|
||||
if err != nil {
|
||||
t.Fatalf("zstd writer: %v", err)
|
||||
}
|
||||
if _, err := zw.Write(data); err != nil {
|
||||
t.Fatalf("zstd write: %v", err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zstd close: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// pipeCompress shells out to a real compressor (bzip2/xz) with data on stdin, since neither
|
||||
// has a compression writer in this module's dependencies; only the decompression side of xz
|
||||
// is production code, this is test-fixture setup only. Skips the calling case when the tool
|
||||
// is not installed, mirroring the tar-availability guard in TestExtractDiskImageFromFile.
|
||||
func pipeCompress(t *testing.T, tool string, args []string, data []byte) []byte {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("%s not found, skipping case", tool)
|
||||
}
|
||||
|
||||
cmd := exec.Command(tool, args...)
|
||||
cmd.Stdin = bytes.NewReader(data)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatalf("%s compress: %v", tool, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestExtractDiskImageFromFile_Compression covers the single-file compressed artifacts that
|
||||
// Talos, Fedora CoreOS and Flatcar publish (.raw.zst, .raw.xz, .qcow2.xz, .img.bz2), none of
|
||||
// which ExtractDiskImageFromFile could import before this fix, plus a genuinely unsupported
|
||||
// suffix. Every case asserts the source artifact survives untouched and that nothing else is
|
||||
// written into its directory, since a regression here would silently duplicate a multi-GiB
|
||||
// image beside itself.
|
||||
func TestExtractDiskImageFromFile_Compression(t *testing.T) {
|
||||
raw := rawFixture(t)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
filename string
|
||||
buildPayload func(t *testing.T) []byte
|
||||
wantErrSubstr string
|
||||
wantPathSuffix string
|
||||
}{
|
||||
{
|
||||
name: "gzip raw payload",
|
||||
filename: "talos.raw.gz",
|
||||
buildPayload: func(t *testing.T) []byte { return gzipBytes(t, raw) },
|
||||
wantPathSuffix: ".raw",
|
||||
},
|
||||
{
|
||||
name: "zstd raw payload",
|
||||
filename: "talos.raw.zst",
|
||||
buildPayload: func(t *testing.T) []byte { return zstdBytes(t, raw) },
|
||||
wantPathSuffix: ".raw",
|
||||
},
|
||||
{
|
||||
name: "bzip2 raw payload",
|
||||
filename: "flatcar.img.bz2",
|
||||
buildPayload: func(t *testing.T) []byte { return pipeCompress(t, "bzip2", []string{"-z", "-c"}, raw) },
|
||||
wantPathSuffix: ".img",
|
||||
},
|
||||
{
|
||||
name: "plain xz raw payload",
|
||||
filename: "talos.raw.xz",
|
||||
buildPayload: func(t *testing.T) []byte { return pipeCompress(t, "xz", []string{"-z", "-c"}, raw) },
|
||||
wantPathSuffix: ".raw",
|
||||
},
|
||||
{
|
||||
name: "qcow2 xz payload converts to raw",
|
||||
filename: "fcos.qcow2.xz",
|
||||
buildPayload: func(t *testing.T) []byte {
|
||||
return pipeCompress(t, "xz", []string{"-z", "-c"}, qcow2FixtureBytes(t))
|
||||
},
|
||||
wantPathSuffix: ".raw",
|
||||
},
|
||||
{
|
||||
name: "genuinely unsupported suffix",
|
||||
filename: "image.raw.7z",
|
||||
buildPayload: func(t *testing.T) []byte { return []byte("not a real archive") },
|
||||
wantErrSubstr: "unsupported filetype",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srcDir := t.TempDir()
|
||||
outDir := t.TempDir()
|
||||
|
||||
payload := tc.buildPayload(t)
|
||||
srcPath := filepath.Join(srcDir, tc.filename)
|
||||
if err := os.WriteFile(srcPath, payload, 0644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
srcBefore, err := os.ReadFile(srcPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source before extract: %v", err)
|
||||
}
|
||||
|
||||
gotPath, err := ExtractDiskImageFromFile(srcPath, outDir)
|
||||
|
||||
if tc.wantErrSubstr != "" {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error containing %q, got none (path=%q)", tc.wantErrSubstr, gotPath)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantErrSubstr) {
|
||||
t.Errorf("err = %v, want substring %q", err, tc.wantErrSubstr)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractDiskImageFromFile(%s) error: %v", tc.filename, err)
|
||||
}
|
||||
if !strings.HasSuffix(gotPath, tc.wantPathSuffix) {
|
||||
t.Errorf("path = %q, want suffix %q", gotPath, tc.wantPathSuffix)
|
||||
}
|
||||
if filepath.Dir(gotPath) != outDir {
|
||||
t.Errorf("extracted path %q not inside tmpdir %q, want beside the tmpdir not the source", gotPath, outDir)
|
||||
}
|
||||
}
|
||||
|
||||
// The source artifact must survive untouched regardless of outcome.
|
||||
srcAfter, err := os.ReadFile(srcPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source after extract: %v", err)
|
||||
}
|
||||
if !bytes.Equal(srcBefore, srcAfter) {
|
||||
t.Error("source artifact was modified during extraction")
|
||||
}
|
||||
|
||||
// Nothing may be written beside the source, in its own directory.
|
||||
entries, err := os.ReadDir(srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read source dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != tc.filename {
|
||||
names := make([]string, len(entries))
|
||||
for i, e := range entries {
|
||||
names[i] = e.Name()
|
||||
}
|
||||
t.Errorf("source dir contains %v, want only %q", names, tc.filename)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecompressToTemp_WritesSparseHoles is the regression test for the on-disk-size blowup
|
||||
// found by verifying this fix live: a Talos raw.zst imported successfully but the decompressed
|
||||
// output was written literally rather than sparsely, costing roughly 60x the disk space of the
|
||||
// qcow2-to-raw path it sits beside. A realistic layout — a small non-zero header, a large zero
|
||||
// run standing in for unused disk space, and a trailing zero run to exercise the final Truncate
|
||||
// — should land on disk as a small fraction of its apparent size.
|
||||
func TestDecompressToTemp_WritesSparseHoles(t *testing.T) {
|
||||
const zeroRun = 8 << 20 // 8 MiB: several multiples of sparseBlockSize, so the test is
|
||||
// meaningful regardless of the exact block size chosen inside decompressToTemp.
|
||||
|
||||
payload := append([]byte("MULGA-SPARSE-FIXTURE-HEADER"), make([]byte, zeroRun)...)
|
||||
|
||||
tmpdir := t.TempDir()
|
||||
outPath, err := decompressToTemp("talos.raw.img", tmpdir, ".img", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("decompressToTemp: %v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatal("decompressed content does not round-trip through the sparse writer")
|
||||
}
|
||||
|
||||
assertSparseOnDisk(t, outPath)
|
||||
}
|
||||
|
||||
// assertSparseOnDisk skips (rather than fails) when the platform does not expose block-count
|
||||
// stat info, or when the test filesystem does not appear to support sparse files at all, since
|
||||
// neither is a real regression in the code under test.
|
||||
func assertSparseOnDisk(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatalf("stat %s: %v", path, err)
|
||||
}
|
||||
apparent := info.Size()
|
||||
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
t.Skip("platform does not expose block-count stat info, skipping sparse-size assertion")
|
||||
}
|
||||
onDisk := stat.Blocks * 512
|
||||
|
||||
if onDisk >= apparent {
|
||||
t.Skip("test filesystem does not appear to support sparse files (on-disk size not below apparent size), skipping")
|
||||
}
|
||||
|
||||
// The zero run should collapse to a small fraction of the apparent size; generous
|
||||
// headroom avoids pinning an exact ratio that would vary with filesystem block size.
|
||||
if onDisk > apparent/4 {
|
||||
t.Errorf("on-disk size = %d bytes for %d apparent bytes, want the zero run to be sparse (< 25%% of apparent)", onDisk, apparent)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecompressXz_WritesSparseHoles covers the plain-.xz path specifically: it shells out to
|
||||
// the xz binary and pipes its stdout through decompressToTemp rather than calling it directly,
|
||||
// so this exercises that piping does not lose the sparse-hole behaviour. .raw.xz is what Talos
|
||||
// and Flatcar actually publish alongside .zst, so this is the format most likely hit in
|
||||
// practice.
|
||||
func TestDecompressXz_WritesSparseHoles(t *testing.T) {
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
t.Skip("xz not found, skipping")
|
||||
}
|
||||
|
||||
const zeroRun = 8 << 20
|
||||
payload := append([]byte("MULGA-SPARSE-FIXTURE-HEADER"), make([]byte, zeroRun)...)
|
||||
compressed := pipeCompress(t, "xz", []string{"-z", "-c"}, payload)
|
||||
|
||||
srcDir := t.TempDir()
|
||||
outDir := t.TempDir()
|
||||
srcPath := filepath.Join(srcDir, "talos.raw.xz")
|
||||
if err := os.WriteFile(srcPath, compressed, 0644); err != nil {
|
||||
t.Fatalf("write source: %v", err)
|
||||
}
|
||||
srcBefore, err := os.ReadFile(srcPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source before decompress: %v", err)
|
||||
}
|
||||
|
||||
outPath, err := decompressXz(srcPath, outDir)
|
||||
if err != nil {
|
||||
t.Fatalf("decompressXz: %v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(outPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read output: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, payload) {
|
||||
t.Fatal("decompressed content does not round-trip through the piped xz decompressor")
|
||||
}
|
||||
|
||||
assertSparseOnDisk(t, outPath)
|
||||
|
||||
srcAfter, err := os.ReadFile(srcPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read source after decompress: %v", err)
|
||||
}
|
||||
if !bytes.Equal(srcBefore, srcAfter) {
|
||||
t.Error("source artifact was modified during xz decompression")
|
||||
}
|
||||
|
||||
entries, err := os.ReadDir(srcDir)
|
||||
if err != nil {
|
||||
t.Fatalf("read source dir: %v", err)
|
||||
}
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(srcPath) {
|
||||
t.Errorf("source dir contains %v, want only %q", entries, filepath.Base(srcPath))
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecompressXz_TruncatedStreamSurfacesStderr guards the io.Pipe plumbing added for the
|
||||
// sparse fix: a Read past the end of a failed xz process must surface xz's own stderr, not a
|
||||
// bare io.EOF that would otherwise look like a clean, silently-truncated decompression.
|
||||
func TestDecompressXz_TruncatedStreamSurfacesStderr(t *testing.T) {
|
||||
if _, err := exec.LookPath("xz"); err != nil {
|
||||
t.Skip("xz not found, skipping")
|
||||
}
|
||||
|
||||
compressed := pipeCompress(t, "xz", []string{"-z", "-c"}, rawFixture(t))
|
||||
truncated := compressed[:len(compressed)/2]
|
||||
|
||||
srcDir := t.TempDir()
|
||||
outDir := t.TempDir()
|
||||
srcPath := filepath.Join(srcDir, "corrupt.raw.xz")
|
||||
if err := os.WriteFile(srcPath, truncated, 0644); err != nil {
|
||||
t.Fatalf("write truncated source: %v", err)
|
||||
}
|
||||
|
||||
_, err := decompressXz(srcPath, outDir)
|
||||
if err == nil {
|
||||
t.Fatal("expected an error decompressing a truncated xz stream")
|
||||
}
|
||||
if err.Error() == io.EOF.Error() {
|
||||
t.Fatalf("error is a bare EOF, want the underlying xz stderr instead: %v", err)
|
||||
}
|
||||
|
||||
msg := strings.ToLower(err.Error())
|
||||
if !strings.Contains(msg, "end of input") && !strings.Contains(msg, "end of file") {
|
||||
t.Errorf("err = %q, want it to surface xz's stderr about the truncated stream", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user