diff --git a/cmd/tar-diff/main.go b/cmd/tar-diff/main.go index 9100eaa..af8f71f 100644 --- a/cmd/tar-diff/main.go +++ b/cmd/tar-diff/main.go @@ -27,7 +27,10 @@ func (p *prefixList) Set(value string) error { var version = flag.Bool("version", false, "Show version") var compressionLevel = flag.Int("compression-level", 3, "zstd compression level") var maxBsdiffSize = flag.Int("max-bsdiff-size", 192, "Max file size in megabytes to consider using bsdiff, or 0 for no limit") +var tmpDir = flag.String("tmp-dir", defaultTmpDir, "Directory for temporary files") +var applyWhiteouts = flag.Bool("apply-whiteouts", false, "Apply docker/OCI whiteout files when analyzing old tar layers") var sourcePrefixes prefixList +var ignoreSourcePrefixes prefixList func closeAndWarn(file *os.File) { if err := file.Close(); err != nil { @@ -37,6 +40,7 @@ func closeAndWarn(file *os.File) { func realMain() int { flag.Var(&sourcePrefixes, "source-prefix", "Only use source files with this path prefix for delta (can be specified multiple times)") + flag.Var(&ignoreSourcePrefixes, "ignore-source-prefix", "Ignore source files with this path prefix for delta (can be specified multiple times)") flag.Usage = func() { _, _ = fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [OPTION] old1.tar.gz [old2.tar.gz ...] new.tar.gz result.tardiff\n", path.Base(os.Args[0])) @@ -93,6 +97,13 @@ func realMain() int { if len(sourcePrefixes) > 0 { options.SetSourcePrefixes(sourcePrefixes) } + if len(ignoreSourcePrefixes) > 0 { + options.SetIgnoreSourcePrefixes(ignoreSourcePrefixes) + } + options.SetTmpDir(*tmpDir) + if *applyWhiteouts { + options.SetApplyWhiteouts(true) + } err = tardiff.Diff(oldFiles, newFile, deltaFile, options) if err != nil { diff --git a/cmd/tar-diff/tmpdir_unix.go b/cmd/tar-diff/tmpdir_unix.go new file mode 100644 index 0000000..f381a8a --- /dev/null +++ b/cmd/tar-diff/tmpdir_unix.go @@ -0,0 +1,5 @@ +//go:build !windows + +package main + +const defaultTmpDir = "/var/tmp" diff --git a/cmd/tar-diff/tmpdir_windows.go b/cmd/tar-diff/tmpdir_windows.go new file mode 100644 index 0000000..f6c096b --- /dev/null +++ b/cmd/tar-diff/tmpdir_windows.go @@ -0,0 +1,7 @@ +//go:build windows + +package main + +import "os" + +var defaultTmpDir = os.TempDir() diff --git a/pkg/tar-diff/analysis.go b/pkg/tar-diff/analysis.go index fb3f168..d960757 100644 --- a/pkg/tar-diff/analysis.go +++ b/pkg/tar-diff/analysis.go @@ -33,9 +33,15 @@ type hardlinkInfo struct { header *tar.Header } +type whiteoutEntry struct { + path string // target path for specific whiteout, or directory prefix for opaque + opaque bool +} + type tarInfo struct { files []tarFileInfo // no size=0 files hardlinks []hardlinkInfo + whiteouts []whiteoutEntry } type targetInfo struct { @@ -47,14 +53,20 @@ type targetInfo struct { type sourceInfo struct { file *tarFileInfo - usedForDelta bool - offset int64 sourceTarFileIndex int + sourcePath string +} + +// Per delta run information about each sourceInfo +type sourceDataInfo struct { + usedForDelta bool + offset int64 } type deltaAnalysis struct { targetInfos []targetInfo sourceInfos []sourceInfo + sourceDataInfos map[*sourceInfo]*sourceDataInfo sourceData *os.File targetInfoByIndex map[int]*targetInfo } @@ -117,7 +129,7 @@ func useTarFile(hdr *tar.Header, cleanPath string) bool { return true } -func analyzeTar(tarMaybeCompressed io.Reader) (*tarInfo, error) { +func analyzeTar(tarMaybeCompressed io.Reader, applyWhiteouts bool) (*tarInfo, error) { tarFile, _, err := compression.AutoDecompress(tarMaybeCompressed) if err != nil { return nil, err @@ -130,6 +142,7 @@ func analyzeTar(tarMaybeCompressed io.Reader) (*tarInfo, error) { files := make([]tarFileInfo, 0) hardlinks := make([]hardlinkInfo, 0) + whiteouts := make([]whiteoutEntry, 0) infoByPath := make(map[string]int) // map from path to index in 'files' rdr := tar.NewReader(tarFile) @@ -162,6 +175,26 @@ func analyzeTar(tarMaybeCompressed io.Reader) (*tarInfo, error) { continue } + // Detect docker/OCI whiteout files + if applyWhiteouts { + basename := path.Base(pathname) + if strings.HasPrefix(basename, ".wh.") { + dir := path.Dir(pathname) + if basename == ".wh..wh..opq" { + if dir == "." { + dir = "" + } else { + dir += "/" + } + whiteouts = append(whiteouts, whiteoutEntry{path: dir, opaque: true}) + } else { + targetName := strings.TrimPrefix(basename, ".wh.") + whiteouts = append(whiteouts, whiteoutEntry{path: path.Join(dir, targetName), opaque: false}) + } + continue + } + } + // If a file is in the archive several times, mark it as overwritten so its not used for delta source if oldIndex, ok := infoByPath[pathname]; ok { files[oldIndex].overwritten = true @@ -199,7 +232,7 @@ func analyzeTar(tarMaybeCompressed io.Reader) (*tarInfo, error) { } } - info := tarInfo{files: files, hardlinks: hardlinks} + info := tarInfo{files: files, hardlinks: hardlinks, whiteouts: whiteouts} return &info, nil } @@ -257,7 +290,7 @@ type indexKey struct { entryIndex int } -func extractDeltaData(tarMaybeCompressedFiles []io.ReadSeeker, sourceByIndex map[indexKey]*sourceInfo, dest *os.File) error { +func extractDeltaData(tarMaybeCompressedFiles []io.ReadSeeker, sourceByIndex map[indexKey]*sourceInfo, sourceDataInfos map[*sourceInfo]*sourceDataInfo, dest *os.File) error { offset := int64(0) for fileIndex, tarMaybeCompressed := range tarMaybeCompressedFiles { @@ -282,11 +315,14 @@ func extractDeltaData(tarMaybeCompressedFiles []io.ReadSeeker, sourceByIndex map return err } info := sourceByIndex[indexKey{fileIndex: fileIndex, entryIndex: index}] - if info != nil && info.usedForDelta { - info.offset = offset - offset += hdr.Size - if _, err := io.Copy(dest, rdr); err != nil { - return err + if info != nil { + sdi := sourceDataInfos[info] + if sdi != nil && sdi.usedForDelta { + sdi.offset = offset + offset += hdr.Size + if _, err := io.Copy(dest, rdr); err != nil { + return err + } } } } @@ -301,24 +337,43 @@ func abs(n int64) int64 { return n } -func buildSourceInfos(oldInfos []*tarInfo) []sourceInfo { +func buildSourceAnalysis(oldInfos []*tarInfo, numOldFiles int, options *Options) *SourceAnalysis { + if options == nil { + options = NewOptions() + } sourceInfos := make([]sourceInfo, 0) pathToFileIndex := make(map[string]int) for fileIdx, oldInfo := range oldInfos { + // Apply whiteouts from this layer to sources from earlier layers + for _, wo := range oldInfo.whiteouts { + if wo.opaque { + for p := range pathToFileIndex { + if hasPathPrefix(p, wo.path) { + delete(pathToFileIndex, p) + } + } + } else { + delete(pathToFileIndex, wo.path) + } + } + for i := range oldInfo.files { file := &oldInfo.files[i] - // Check if any path from this file conflicts with existing files + // Remove any paths from this file that conflict with + // existing sources from earlier layers for _, p := range file.paths { - if existingIdx, exists := pathToFileIndex[p]; exists { - sourceInfos[existingIdx].file.overwritten = true - } + delete(pathToFileIndex, p) } - // Add the primary path of this file (which is the one used as delta source) currentFileIndex := len(sourceInfos) - pathToFileIndex[file.paths[0]] = currentFileIndex + + // Register all paths of this file so whiteouts and + // overwrites from later layers can find them + for _, p := range file.paths { + pathToFileIndex[p] = currentFileIndex + } sourceInfos = append(sourceInfos, sourceInfo{ file: file, @@ -327,7 +382,57 @@ func buildSourceInfos(oldInfos []*tarInfo) []sourceInfo { } } - return sourceInfos + // Now that all layers have been processed and pathToFileIndex + // reflects the final state, compute sourcePath for each source + // and build the lookup maps + sourceBySha1 := make(map[string]*sourceInfo) + sourceByPath := make(map[string]*sourceInfo) + sourceByIndex := make(map[indexKey]*sourceInfo) + + for i := range sourceInfos { + s := &sourceInfos[i] + + // Pick the first surviving path that passes prefix filters + for _, p := range s.file.paths { + if idx, exists := pathToFileIndex[p]; !exists || idx != i { + continue + } + if isIgnoredPrefix(p, options.ignoreSourcePrefixes) { + continue + } + if !matchesAnyPrefix(p, options.sourcePrefixes) { + continue + } + s.sourcePath = p + break + } + + if s.sourcePath == "" { + s.file.overwritten = true + continue + } + + sourceBySha1[s.file.sha1] = s + for _, p := range s.file.paths { + sourceByPath[p] = s + } + sourceByIndex[indexKey{fileIndex: s.sourceTarFileIndex, entryIndex: s.file.index}] = s + } + + return &SourceAnalysis{ + sourceInfos: sourceInfos, + sourceBySha1: sourceBySha1, + sourceByPath: sourceByPath, + sourceByIndex: sourceByIndex, + numOldFiles: numOldFiles, + } +} + +func hasPathPrefix(s, prefix string) bool { + if !strings.HasPrefix(s, prefix) { + return false + } + return len(s) == len(prefix) || prefix == "" || strings.HasSuffix(prefix, "/") || s[len(prefix)] == '/' } func matchesAnyPrefix(path string, prefixes []string) bool { @@ -335,22 +440,27 @@ func matchesAnyPrefix(path string, prefixes []string) bool { return true } for _, prefix := range prefixes { - if strings.HasPrefix(path, prefix) { + if hasPathPrefix(path, prefix) { return true } } return false } -func isDeltaSourceCandidate(s *sourceInfo, options *Options) bool { - if s.file.overwritten { - return false +func isIgnoredPrefix(path string, prefixes []string) bool { + for _, prefix := range prefixes { + if hasPathPrefix(path, prefix) { + return true + } } - primaryPath := s.file.paths[0] - return matchesAnyPrefix(primaryPath, options.sourcePrefixes) + return false } -func findFuzzyDeltaSource(sourceInfos []sourceInfo, targetFile *tarFileInfo, options *Options) *sourceInfo { +func isDeltaSourceCandidate(s *sourceInfo) bool { + return s.sourcePath != "" +} + +func findFuzzyDeltaSource(sourceInfos []sourceInfo, targetFile *tarFileInfo) *sourceInfo { // Check for moved (first) or renamed (second) versions for fuzzy := 0; fuzzy < 2; fuzzy++ { var source *sourceInfo @@ -358,7 +468,7 @@ func findFuzzyDeltaSource(sourceInfos []sourceInfo, targetFile *tarFileInfo, opt s := &sourceInfos[j] // Skip files that we're not allowed to use - if !isDeltaSourceCandidate(s, options) { + if !isDeltaSourceCandidate(s) { continue } // Skip files that make no sense to delta (like compressed files) @@ -387,36 +497,20 @@ func findFuzzyDeltaSource(sourceInfos []sourceInfo, targetFile *tarFileInfo, opt return nil } -func analyzeForDelta(oldInfos []*tarInfo, newTar *tarInfo, oldFiles []io.ReadSeeker, options *Options) (*deltaAnalysis, error) { +func analyzeForDelta(sources *SourceAnalysis, newTar *tarInfo, oldFiles []io.ReadSeeker, options *Options) (*deltaAnalysis, error) { if options == nil { options = NewOptions() } - sourceInfos := buildSourceInfos(oldInfos) - - sourceBySha1 := make(map[string]*sourceInfo) - sourceByPath := make(map[string]*sourceInfo) - sourceByIndex := make(map[indexKey]*sourceInfo) - for i := range sourceInfos { - s := &sourceInfos[i] - if !isDeltaSourceCandidate(s, options) { - continue - } - sourceBySha1[s.file.sha1] = s - for _, p := range s.file.paths { - sourceByPath[p] = s - } - sourceByIndex[indexKey{fileIndex: s.sourceTarFileIndex, entryIndex: s.file.index}] = s - } - targetInfos := make([]targetInfo, 0, len(newTar.files)+len(newTar.hardlinks)) + sourceDataInfos := make(map[*sourceInfo]*sourceDataInfo) for i := range newTar.files { file := &newTar.files[i] // First look for exact content match usedForDelta := false var source *sourceInfo - sha1Source := sourceBySha1[file.sha1] + sha1Source := sources.sourceBySha1[file.sha1] // If same sha1 and size, use original total size if sha1Source != nil && file.size == sha1Source.file.size { source = sha1Source @@ -427,7 +521,7 @@ func analyzeForDelta(oldInfos []*tarInfo, newTar *tarInfo, oldFiles []io.ReadSee // Check if any of the target file's paths match a source file var s *sourceInfo for _, p := range file.paths { - if matchedSource := sourceByPath[p]; matchedSource != nil { + if matchedSource := sources.sourceByPath[p]; matchedSource != nil { s = matchedSource break } @@ -437,7 +531,7 @@ func analyzeForDelta(oldInfos []*tarInfo, newTar *tarInfo, oldFiles []io.ReadSee usedForDelta = true source = s } else { - source = findFuzzyDeltaSource(sourceInfos, file, options) + source = findFuzzyDeltaSource(sources.sourceInfos, file) if source != nil { usedForDelta = true } @@ -446,7 +540,12 @@ func analyzeForDelta(oldInfos []*tarInfo, newTar *tarInfo, oldFiles []io.ReadSee var rollsumMatches *rollsumMatches if source != nil { - source.usedForDelta = source.usedForDelta || usedForDelta + sdi := sourceDataInfos[source] + if sdi == nil { + sdi = &sourceDataInfo{} + sourceDataInfos[source] = sdi + } + sdi.usedForDelta = sdi.usedForDelta || usedForDelta if usedForDelta { rollsumMatches = computeRollsumMatches(source.file.blobs, file.blobs) @@ -469,16 +568,16 @@ func analyzeForDelta(oldInfos []*tarInfo, newTar *tarInfo, oldFiles []io.ReadSee targetInfoByIndex[hl.index] = &targetInfos[len(targetInfos)-1] } - tmpfile, err := os.CreateTemp(os.TempDir(), "tar-diff-") + tmpfile, err := os.CreateTemp(options.tmpDir, "tar-diff-") if err != nil { return nil, err } - err = extractDeltaData(oldFiles, sourceByIndex, tmpfile) + err = extractDeltaData(oldFiles, sources.sourceByIndex, sourceDataInfos, tmpfile) if err != nil { _ = os.Remove(tmpfile.Name()) return nil, err } - return &deltaAnalysis{targetInfos: targetInfos, targetInfoByIndex: targetInfoByIndex, sourceInfos: sourceInfos, sourceData: tmpfile}, nil + return &deltaAnalysis{targetInfos: targetInfos, targetInfoByIndex: targetInfoByIndex, sourceInfos: sources.sourceInfos, sourceDataInfos: sourceDataInfos, sourceData: tmpfile}, nil } diff --git a/pkg/tar-diff/analysis_test.go b/pkg/tar-diff/analysis_test.go index e896a83..9258920 100644 --- a/pkg/tar-diff/analysis_test.go +++ b/pkg/tar-diff/analysis_test.go @@ -59,7 +59,7 @@ func TestAnalyzeTar_Hardlinks(t *testing.T) { t.Fatalf("Failed to create test tar: %v", err) } - info, err := analyzeTar(tarFile) + info, err := analyzeTar(tarFile, false) if err != nil { t.Fatalf("analyzeTar failed: %v", err) } @@ -99,7 +99,7 @@ func TestAnalyzeTar_MultipleHardlinks(t *testing.T) { t.Fatalf("Failed to create test tar: %v", err) } - info, err := analyzeTar(tarFile) + info, err := analyzeTar(tarFile, false) if err != nil { t.Fatalf("analyzeTar failed: %v", err) } @@ -133,7 +133,7 @@ func TestAnalyzeTar_DuplicateFilesAndHardlinks(t *testing.T) { t.Fatalf("Failed to create test tar: %v", err) } - info, err := analyzeTar(tarFile) + info, err := analyzeTar(tarFile, false) if err != nil { t.Fatalf("analyzeTar failed: %v", err) } @@ -174,12 +174,12 @@ func TestAnalyzeForDelta_HardlinksInTargetInfo(t *testing.T) { t.Fatalf("newTar.Seek: %v", err) } - oldInfo, err := analyzeTar(oldTar) + oldInfo, err := analyzeTar(oldTar, false) if err != nil { t.Fatalf("analyzeTar (old) failed: %v", err) } - newInfo, err := analyzeTar(newTar) + newInfo, err := analyzeTar(newTar, false) if err != nil { t.Fatalf("analyzeTar (new) failed: %v", err) } @@ -189,7 +189,7 @@ func TestAnalyzeForDelta_HardlinksInTargetInfo(t *testing.T) { t.Fatalf("oldTar.Seek: %v", err) } - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{oldTar}, nil) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, nil), newInfo, []io.ReadSeeker{oldTar}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } @@ -226,7 +226,7 @@ func TestAnalyzeTar_HardlinksAddMultiplePaths(t *testing.T) { t.Fatalf("Failed to create test tar: %v", err) } - info, err := analyzeTar(tarFile) + info, err := analyzeTar(tarFile, false) if err != nil { t.Fatalf("analyzeTar failed: %v", err) } @@ -286,12 +286,12 @@ func TestAnalyzeForDelta_MatchViaHardlinkPath(t *testing.T) { t.Fatalf("newTar.Seek: %v", err) } - oldInfo, err := analyzeTar(oldTar) + oldInfo, err := analyzeTar(oldTar, false) if err != nil { t.Fatalf("analyzeTar (old) failed: %v", err) } - newInfo, err := analyzeTar(newTar) + newInfo, err := analyzeTar(newTar, false) if err != nil { t.Fatalf("analyzeTar (new) failed: %v", err) } @@ -300,7 +300,7 @@ func TestAnalyzeForDelta_MatchViaHardlinkPath(t *testing.T) { t.Fatalf("oldTar.Seek: %v", err) } - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{oldTar}, nil) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, nil), newInfo, []io.ReadSeeker{oldTar}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } diff --git a/pkg/tar-diff/diff.go b/pkg/tar-diff/diff.go index 6464cbd..221e338 100644 --- a/pkg/tar-diff/diff.go +++ b/pkg/tar-diff/diff.go @@ -58,7 +58,8 @@ func (g *deltaGenerator) copyN(n int64) error { // Read back part of the stored data for the source file func (g *deltaGenerator) readSourceData(source *sourceInfo, offset int64, size int64) ([]byte, error) { - _, err := g.analysis.sourceData.Seek(int64(source.offset+offset), 0) + sdi := g.analysis.sourceDataInfos[source] + _, err := g.analysis.sourceData.Seek(int64(sdi.offset+offset), 0) if err != nil { return nil, err } @@ -71,7 +72,7 @@ func (g *deltaGenerator) generateForFileWithBsdiff(info *targetInfo) error { file := info.file source := info.source - err := g.deltaWriter.SetCurrentFile(source.file.paths[0]) + err := g.deltaWriter.SetCurrentFile(info.source.sourcePath) if err != nil { return err } @@ -105,7 +106,7 @@ func (g *deltaGenerator) generateForFileWithrollsums(info *targetInfo) error { matches := info.rollsumMatches.matches pos := int64(0) - err := g.deltaWriter.SetCurrentFile(source.file.paths[0]) + err := g.deltaWriter.SetCurrentFile(info.source.sourcePath) if err != nil { return err } @@ -161,7 +162,7 @@ func (g *deltaGenerator) generateForFile(info *targetInfo) error { switch { case sourceFile.sha1 == file.sha1 && sourceFile.size == file.size: // Reuse exact file from old tar - if err := g.deltaWriter.WriteOldFile(sourceFile.paths[0], uint64(sourceFile.size)); err != nil { + if err := g.deltaWriter.WriteOldFile(info.source.sourcePath, uint64(sourceFile.size)); err != nil { return err } @@ -260,9 +261,12 @@ func generateDelta(newFile io.ReadSeeker, deltaFile io.Writer, analysis *deltaAn // Options configures the behavior of the diff operation. type Options struct { - compressionLevel int - maxBsdiffSize int64 - sourcePrefixes []string + compressionLevel int + maxBsdiffSize int64 + sourcePrefixes []string + ignoreSourcePrefixes []string + tmpDir string + applyWhiteouts bool } // SetCompressionLevel sets the compression level for the output diff file. @@ -281,70 +285,128 @@ func (o *Options) SetSourcePrefixes(prefixes []string) { o.sourcePrefixes = prefixes } +// SetTmpDir sets the directory for temporary files. Defaults to os.TempDir(). +func (o *Options) SetTmpDir(dir string) { + o.tmpDir = dir +} + +// SetIgnoreSourcePrefixes sets path prefixes to exclude from delta sources. +// Files whose paths all match one of these prefixes will not be used as delta sources. +// If a file has multiple names (hardlinks), any non-ignored name makes the file usable. +func (o *Options) SetIgnoreSourcePrefixes(prefixes []string) { + o.ignoreSourcePrefixes = prefixes +} + +// SetApplyWhiteouts enables docker/OCI-style whiteout processing when analyzing +// old tar layers. Whiteout files (.wh. and .wh..wh..opq) in upper layers +// remove matching paths from lower layers, so the delta sources reflect the +// merged container image rather than individual layers. +func (o *Options) SetApplyWhiteouts(apply bool) { + o.applyWhiteouts = apply +} + // NewOptions creates a new Options struct with default values. func NewOptions() *Options { return &Options{ - compressionLevel: 3, - maxBsdiffSize: defaultMaxBsdiffSize, - sourcePrefixes: nil, + compressionLevel: 3, + maxBsdiffSize: defaultMaxBsdiffSize, + sourcePrefixes: nil, + ignoreSourcePrefixes: nil, } } -// Diff creates a binary difference between a set of tar archives and a new tar archive -// oldTarFiles contains one or more old tar files, in extraction order -func Diff(oldTarFiles []io.ReadSeeker, newTarFile io.ReadSeeker, diffFile io.Writer, options *Options) error { +// SourceAnalysis contains information about pre-analyzed delta sources. +// It can be reused across multiple DiffWithSources calls, including +// concurrent calls from different goroutines, as long as each call +// provides its own independent set of old tar readers. +type SourceAnalysis struct { + sourceInfos []sourceInfo + sourceBySha1 map[string]*sourceInfo + sourceByPath map[string]*sourceInfo + sourceByIndex map[indexKey]*sourceInfo + numOldFiles int +} +// AnalyzeSources pre-computes analysis of one or more old tar files +// that can be reused across multiple DiffWithSources operations. +// oldTarFiles contains one or more old tar files, in extraction order. +// The readers are only used during this call and are not retained. +func AnalyzeSources(oldTarFiles []io.ReadSeeker, options *Options) (*SourceAnalysis, error) { if options == nil { options = NewOptions() } if len(oldTarFiles) == 0 { - return fmt.Errorf("at least one old tar file is required") + return nil, fmt.Errorf("at least one old tar file is required") } - // First analyze all tarfiles by themselves oldInfos := make([]*tarInfo, len(oldTarFiles)) for i, oldTarFile := range oldTarFiles { - oldInfo, err := analyzeTar(oldTarFile) + oldInfo, err := analyzeTar(oldTarFile, options.applyWhiteouts) if err != nil { - return err + return nil, err } oldInfos[i] = oldInfo } - newInfo, err := analyzeTar(newTarFile) - if err != nil { - return err + return buildSourceAnalysis(oldInfos, len(oldTarFiles), options), nil +} + +// DiffWithSources creates a binary difference using a pre-computed +// SourceAnalysis. The oldTarFiles must correspond to the same tar +// files (in the same order) that were passed to AnalyzeSources. If +// they are independent readers then that allows concurrent calls from +// multiple goroutines. +func DiffWithSources(sources *SourceAnalysis, oldTarFiles []io.ReadSeeker, newTarFile io.ReadSeeker, diffFile io.Writer, options *Options) error { + if sources == nil { + return fmt.Errorf("sources cannot be nil") } - // Reset tar.gz for re-reading - for _, oldTarFile := range oldTarFiles { - _, err = oldTarFile.Seek(0, 0) - if err != nil { - return err - } + if options == nil { + options = NewOptions() + } + + if len(oldTarFiles) != sources.numOldFiles { + return fmt.Errorf("expected %d old tar files, got %d", sources.numOldFiles, len(oldTarFiles)) } - _, err = newTarFile.Seek(0, 0) + + newInfo, err := analyzeTar(newTarFile, false) if err != nil { return err } - // Compare new and old for delta information - analysis, err := analyzeForDelta(oldInfos, newInfo, oldTarFiles, options) + if _, err := newTarFile.Seek(0, 0); err != nil { + return err + } + + analysis, err := analyzeForDelta(sources, newInfo, oldTarFiles, options) if err != nil { return err } defer func() { if err := analysis.Close(); err != nil { - log.Printf("close tar file: %v", err) + log.Printf("close analysis: %v", err) } }() - // Actually create the delta - if err := generateDelta(newTarFile, diffFile, analysis, options); err != nil { + return generateDelta(newTarFile, diffFile, analysis, options) +} + +// Diff creates a binary difference between a set of tar archives and a new tar archive. +// oldTarFiles contains one or more old tar files, in extraction order. +func Diff(oldTarFiles []io.ReadSeeker, newTarFile io.ReadSeeker, diffFile io.Writer, options *Options) error { + sources, err := AnalyzeSources(oldTarFiles, options) + if err != nil { return err } - return nil + // Reset old files after AnalyzeSources read them + for _, oldTarFile := range oldTarFiles { + if _, err := oldTarFile.Seek(0, 0); err != nil { + return err + } + } + + return DiffWithSources(sources, oldTarFiles, newTarFile, diffFile, options) } diff --git a/pkg/tar-diff/diff_test.go b/pkg/tar-diff/diff_test.go index 946fe4f..ab4cc83 100644 --- a/pkg/tar-diff/diff_test.go +++ b/pkg/tar-diff/diff_test.go @@ -34,12 +34,12 @@ func TestGenerateDelta_Hardlinks(t *testing.T) { t.Fatalf("oldTar.Seek: %v", err) } - newInfo, err := analyzeTar(newTar) + newInfo, err := analyzeTar(newTar, false) if err != nil { t.Fatalf("analyzeTar (new) failed: %v", err) } - oldInfo, err := analyzeTar(oldTar) + oldInfo, err := analyzeTar(oldTar, false) if err != nil { t.Fatalf("analyzeTar (old) failed: %v", err) } @@ -48,7 +48,7 @@ func TestGenerateDelta_Hardlinks(t *testing.T) { if _, err := oldTar.Seek(0, 0); err != nil { t.Fatalf("oldTar.Seek: %v", err) } - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{oldTar}, nil) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, nil), newInfo, []io.ReadSeeker{oldTar}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } @@ -114,12 +114,12 @@ func TestGenerateDelta_MixedHardlinksAndDuplicates(t *testing.T) { t.Fatalf("oldTar.Seek: %v", err) } - newInfo, err := analyzeTar(newTar) + newInfo, err := analyzeTar(newTar, false) if err != nil { t.Fatalf("analyzeTar (new) failed: %v", err) } - oldInfo, err := analyzeTar(oldTar) + oldInfo, err := analyzeTar(oldTar, false) if err != nil { t.Fatalf("analyzeTar (old) failed: %v", err) } @@ -136,7 +136,7 @@ func TestGenerateDelta_MixedHardlinksAndDuplicates(t *testing.T) { if _, err := oldTar.Seek(0, 0); err != nil { t.Fatalf("oldTar.Seek: %v", err) } - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{oldTar}, nil) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, nil), newInfo, []io.ReadSeeker{oldTar}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } diff --git a/pkg/tar-diff/multifile_test.go b/pkg/tar-diff/multifile_test.go index e2d334b..69f9812 100644 --- a/pkg/tar-diff/multifile_test.go +++ b/pkg/tar-diff/multifile_test.go @@ -25,17 +25,17 @@ func TestBuildSourceInfos(t *testing.T) { t.Fatalf("Failed to create tar2: %v", err) } - info1, err := analyzeTar(tar1) + info1, err := analyzeTar(tar1, false) if err != nil { t.Fatalf("Failed to analyze tar1: %v", err) } - info2, err := analyzeTar(tar2) + info2, err := analyzeTar(tar2, false) if err != nil { t.Fatalf("Failed to analyze tar2: %v", err) } - sourceInfos := buildSourceInfos([]*tarInfo{info1, info2}) + sourceInfos := buildSourceAnalysis([]*tarInfo{info1, info2}, 2, NewOptions()).sourceInfos // Should have 3 files total (file1, file2-orig, file2-override, file3) // But file2-orig should be marked as overwritten @@ -101,17 +101,17 @@ func TestBuildSourceInfos_HardlinkConflicts(t *testing.T) { t.Fatalf("Failed to create tar2: %v", err) } - info1, err := analyzeTar(tar1) + info1, err := analyzeTar(tar1, false) if err != nil { t.Fatalf("Failed to analyze tar1: %v", err) } - info2, err := analyzeTar(tar2) + info2, err := analyzeTar(tar2, false) if err != nil { t.Fatalf("Failed to analyze tar2: %v", err) } - sourceInfos := buildSourceInfos([]*tarInfo{info1, info2}) + sourceInfos := buildSourceAnalysis([]*tarInfo{info1, info2}, 2, NewOptions()).sourceInfos // Should have 4 files (two from each layer) if len(sourceInfos) != 4 { diff --git a/pkg/tar-diff/prefix_filter_test.go b/pkg/tar-diff/prefix_filter_test.go index a63b0e4..d301f13 100644 --- a/pkg/tar-diff/prefix_filter_test.go +++ b/pkg/tar-diff/prefix_filter_test.go @@ -52,7 +52,7 @@ func setupPrefixFilterTestData(t *testing.T) (oldTar io.ReadSeeker, oldTarInfo * t.Fatalf("Failed to create new tar: %v", err) } - oldTarInfo, err = analyzeTar(oldTar) + oldTarInfo, err = analyzeTar(oldTar, false) if err != nil { t.Fatalf("Failed to analyze oldTar: %v", err) } @@ -60,7 +60,7 @@ func setupPrefixFilterTestData(t *testing.T) (oldTar io.ReadSeeker, oldTarInfo * t.Fatalf("oldTar.Seek: %v", err) } - newInfo, err = analyzeTar(newTar) + newInfo, err = analyzeTar(newTar, false) if err != nil { t.Fatalf("Failed to analyze new tar: %v", err) } @@ -77,7 +77,7 @@ func TestDiff_SourcePrefix(t *testing.T) { options := NewOptions() options.SetSourcePrefixes([]string{"blobs/"}) - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{old}, options) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, options), newInfo, []io.ReadSeeker{old}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } @@ -102,7 +102,7 @@ func TestDiff_SourcePrefix(t *testing.T) { // Should have a source (matches prefix) if source == nil { t.Error("blobs/sha256/abc123 should have a source (matches prefix)") - } else if !source.usedForDelta { + } else if sdi := analysis.sourceDataInfos[source]; sdi == nil || !sdi.usedForDelta { t.Error("blobs/sha256/abc123 source should be usedForDelta") } @@ -121,7 +121,7 @@ func TestDiff_SourceMultiplePrefixes(t *testing.T) { options := NewOptions() options.SetSourcePrefixes([]string{"blobs/", "config/"}) - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{old}, options) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, options), newInfo, []io.ReadSeeker{old}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } @@ -157,7 +157,7 @@ func TestDiff_NoPrefixFilter(t *testing.T) { old, oldInfo, _, newInfo := setupPrefixFilterTestData(t) // No prefix filter (default) - pass nil for default options - analysis, err := analyzeForDelta([]*tarInfo{oldInfo}, newInfo, []io.ReadSeeker{old}, nil) + analysis, err := analyzeForDelta(buildSourceAnalysis([]*tarInfo{oldInfo}, 1, nil), newInfo, []io.ReadSeeker{old}, nil) if err != nil { t.Fatalf("analyzeForDelta failed: %v", err) } diff --git a/tests/test.sh b/tests/test.sh index 7749ce0..cf0f88c 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -31,6 +31,11 @@ ln_hard(){ TEST_DIR=$(mktemp -d /tmp/test-tardiff-XXXXXX) +cleanup () { + rm -rf $TEST_DIR +} +trap cleanup EXIT + create_orig () { DIR=$1 @@ -92,6 +97,8 @@ create_tar () { compress_tar $FILE } +# --- Basic single-layer test --- + create_orig $TEST_DIR/orig create_tar $TEST_DIR/orig.tar $TEST_DIR/orig @@ -109,7 +116,101 @@ cmp $TEST_DIR/reconstructed.tar $TEST_DIR/modified.tar echo OK -cleanup () { - rm -rf $TEST_DIR +# --- Multi-layer test (simulating OCI image layers with whiteouts) --- + +create_multi_orig1 () { + DIR=$1 + mkdir -p $DIR/data/subdir + echo "base-file1" > $DIR/data/file1.txt + echo "base-file2" > $DIR/data/file2.txt + dd if=/dev/urandom of=$DIR/data/deleted.txt bs=1024 count=64 2>/dev/null + dd if=/dev/urandom of=$DIR/data/subdir/a.txt bs=1024 count=64 2>/dev/null + dd if=/dev/urandom of=$DIR/data/subdir/b.txt bs=1024 count=64 2>/dev/null } -trap cleanup EXIT + +create_multi_orig2 () { + DIR=$1 + mkdir -p $DIR/data/subdir + echo "layer2-file2-override" > $DIR/data/file2.txt + echo "layer2-file3" > $DIR/data/file3.txt + touch $DIR/data/.wh.deleted.txt + touch $DIR/data/subdir/.wh..wh..opq + echo "new-subdir-file" > $DIR/data/subdir/c.txt +} + +create_multi_modified () { + DIR=$1 + ORIG1_DIR=$2 + mkdir -p $DIR/data/subdir + echo "base-file1" > $DIR/data/file1.txt + echo "layer2-file2-override" > $DIR/data/file2.txt + echo "layer3-file3-modified" > $DIR/data/file3.txt + echo "new-subdir-file" > $DIR/data/subdir/c.txt + cp $ORIG1_DIR/data/deleted.txt $DIR/data/deleted.txt + echo "extra" >> $DIR/data/deleted.txt + cp $ORIG1_DIR/data/subdir/a.txt $DIR/data/subdir/a.txt + echo "extra" >> $DIR/data/subdir/a.txt +} + +echo "Testing multi-layer tar diff (simulating OCI image layers)" + +create_multi_orig1 $TEST_DIR/multi_orig1 +create_tar $TEST_DIR/multi_orig1.tar $TEST_DIR/multi_orig1 + +create_multi_orig2 $TEST_DIR/multi_orig2 +create_tar $TEST_DIR/multi_orig2.tar $TEST_DIR/multi_orig2 + +create_multi_modified $TEST_DIR/multi_modified $TEST_DIR/multi_orig1 +create_tar $TEST_DIR/multi_modified.tar $TEST_DIR/multi_modified + +# Build combined directories for patching +mkdir -p $TEST_DIR/combined-full +tar xf $TEST_DIR/multi_orig1.tar -C $TEST_DIR/combined-full +tar xf $TEST_DIR/multi_orig2.tar -C $TEST_DIR/combined-full + +mkdir -p $TEST_DIR/combined-merged +tar xf $TEST_DIR/multi_orig1.tar -C $TEST_DIR/combined-merged +rm -f $TEST_DIR/combined-merged/data/deleted.txt +rm -rf $TEST_DIR/combined-merged/data/subdir +mkdir -p $TEST_DIR/combined-merged/data/subdir +tar xf $TEST_DIR/multi_orig2.tar -C $TEST_DIR/combined-merged --exclude='*/.wh.*' + +# Test 1: without --apply-whiteouts +echo "Test 1: multi-layer diff without --apply-whiteouts" +./tar-diff $TEST_DIR/multi_orig1.tar $TEST_DIR/multi_orig2.tar $TEST_DIR/multi_modified.tar $TEST_DIR/no-whiteout.tardiff + +echo "Applying tardiff (without whiteouts, full combined directory)" +./tar-patch $TEST_DIR/no-whiteout.tardiff $TEST_DIR/combined-full $TEST_DIR/reconstructed1.tar + +echo "Verifying reconstruction" +cmp $TEST_DIR/reconstructed1.tar $TEST_DIR/multi_modified.tar + +echo "Verifying that applying without --apply-whiteouts fails on merged directory" +if ./tar-patch $TEST_DIR/no-whiteout.tardiff $TEST_DIR/combined-merged $TEST_DIR/should-fail.tar 2>/dev/null; then + if cmp -s $TEST_DIR/should-fail.tar $TEST_DIR/multi_modified.tar; then + echo " (diff did not reference whited-out files, reconstruction still correct)" + else + echo " FAIL: reconstruction was incorrect" + exit 1 + fi +else + echo " OK: tar-patch correctly failed (missing whited-out source files)" +fi + +# Test 2: with --apply-whiteouts +echo "Test 2: multi-layer diff with --apply-whiteouts" +./tar-diff --apply-whiteouts $TEST_DIR/multi_orig1.tar $TEST_DIR/multi_orig2.tar $TEST_DIR/multi_modified.tar $TEST_DIR/with-whiteout.tardiff + +echo "Applying tardiff (with whiteouts, merged directory)" +./tar-patch $TEST_DIR/with-whiteout.tardiff $TEST_DIR/combined-merged $TEST_DIR/reconstructed2.tar + +echo "Verifying reconstruction" +cmp $TEST_DIR/reconstructed2.tar $TEST_DIR/multi_modified.tar + +echo "Applying tardiff (with whiteouts, full combined directory)" +./tar-patch $TEST_DIR/with-whiteout.tardiff $TEST_DIR/combined-full $TEST_DIR/reconstructed3.tar + +echo "Verifying reconstruction" +cmp $TEST_DIR/reconstructed3.tar $TEST_DIR/multi_modified.tar + +echo "All tests OK"