tarfs: rewrite to support random access archives

This change adds support for some schemes of encoding a table of
contents and allowing individual files to be accessed independently.
It breaks the API by adding a "size" parameter (like the archive/zip
package) but attempts to autodetect when given a nonsense value.

Signed-off-by: Hank Donnay <[email protected]>

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/rs/zerolog v1.30.0
 	github.com/ulikunitz/xz v0.5.11
 	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/metric v1.19.0
 	go.opentelemetry.io/otel/trace v1.19.0
 	golang.org/x/crypto v0.14.0
 	golang.org/x/sync v0.4.0
@@ -56,7 +57,6 @@ require (
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect

layer.go

@@ -110,13 +110,14 @@ func (l *Layer) Init(ctx context.Context, desc *LayerDescription, r io.ReaderAt)
 		`application/vnd.oci.image.layer.nondistributable.v1.tar`,
 		`application/vnd.oci.image.layer.nondistributable.v1.tar+gzip`,
 		`application/vnd.oci.image.layer.nondistributable.v1.tar+zstd`:
-		sys, err := tarfs.New(r)
+		sys, err := tarfs.New(ctx, r, -1, nil)
 		switch {
 		case errors.Is(err, nil):
 		default:
 			return fmt.Errorf("claircore: layer %v: unable to create fs.FS: %w", desc.Digest, err)
 		}
 		l.sys = sys
+		l.cleanup = append(l.cleanup, sys)
 	default:
 		return fmt.Errorf("claircore: layer %v: unknown MediaType %q", desc.Digest, desc.MediaType)
 	}

pkg/tarfs/file.go

@@ -1,84 +1,141 @@
 package tarfs
 
 import (
-	"archive/tar"
 	"io"
 	"io/fs"
-	"path/filepath"
+	"path"
 	"strings"
+	"time"
 )
 
-var _ fs.File = (*file)(nil)
-
-// File implements fs.File.
-type file struct {
-	h *tar.Header
-	r *tar.Reader
+// Entry is an entry describing a file or a file chunk.
+//
+// This is the concrete type backing [fs.FileInfo] interfaces returned by
+// this package.
+type Entry struct {
+	Xattrs      map[string]string `json:"xattrs"`
+	Type        string            `json:"type"`
+	Name        string            `json:"name"` // NB This is actually the path.
+	Linkname    string            `json:"linkName"`
+	Digest      string            `json:"digest"`
+	ChunkDigest string            `json:"chunkDigest"`
+	UserName    string            `json:"userName"`  // eStargz only
+	GroupName   string            `json:"groupName"` // eStargz only
+	ModTime     time.Time         `json:"modtime"`
+	AccessTime  time.Time         `json:"accesstime"` // Zstd chunked only
+	ChangeTime  time.Time         `json:"changetime"` // Zstd chunked only
+	Mode        int64             `json:"mode"`
+	Size        int64             `json:"size"`
+	Devmajor    int64             `json:"devMajor"`
+	Devminor    int64             `json:"devMinor"`
+	Offset      int64             `json:"offset"`
+	EndOffset   int64             `json:"endOffset"` // Zstd chunked only
+	ChunkSize   int64             `json:"chunkSize"`
+	ChunkOffset int64             `json:"chunkOffset"`
+	UID         int               `json:"uid"`
+	GID         int               `json:"gid"`
 }
 
-func (f *file) Close() error {
-	return nil
-}
+// Entry types.
+const (
+	typeDir      = `dir`
+	typeReg      = `reg`
+	typeSymlink  = `symlink`
+	typeHardlink = `hardlink`
+	typeChar     = `char`
+	typeBlock    = `block`
+	typeFifo     = `fifo`
+	typeChunk    = `chunk`
+)
 
-func (f *file) Read(b []byte) (int, error) {
-	return f.r.Read(b)
+// NewEntryDir returns a new Entry describing a directory at the path "n".
+func newEntryDir(n string) Entry {
+	return Entry{
+		Name: n,
+		Mode: int64(fs.ModeDir | 0o644),
+		Type: typeDir,
+	}
 }
 
-func (f *file) Stat() (fs.FileInfo, error) {
-	return f.h.FileInfo(), nil
+// SortDirent returns a function suitable to pass to [sort.Slice] as a "cmp"
+// function.
+//
+// This is needed because the [io/fs] interfaces specify that [fs.DirEntry]
+// slices returned by the ReadDir methods are sorted lexically.
+func sortDirent(s []fs.DirEntry) func(i, j int) bool {
+	return func(i, j int) bool {
+		return strings.Compare(s[i].Name(), s[j].Name()) == -1
+	}
 }
 
-var _ fs.ReadDirFile = (*dir)(nil)
+// Dirent implements [fs.DirEntry] using a backing [*Entry].
+type dirent struct{ *Entry }
 
-// Dir implements fs.ReadDirFile.
-type dir struct {
-	h   *tar.Header
-	es  []fs.DirEntry
-	pos int
-}
+// Interface assertion for dirent.
+var _ fs.DirEntry = dirent{}
 
-func (*dir) Close() error                 { return nil }
-func (*dir) Read(_ []byte) (int, error)   { return 0, io.EOF }
-func (d *dir) Stat() (fs.FileInfo, error) { return d.h.FileInfo(), nil }
-func (d *dir) ReadDir(n int) ([]fs.DirEntry, error) {
-	es := d.es[d.pos:]
-	if len(es) == 0 {
-		if n == -1 {
-			return nil, nil
-		}
-		return nil, io.EOF
-	}
-	end := min(len(es), n)
-	if n == -1 {
-		end = len(es)
-	}
-	d.pos += end
-	return es[:end], nil
+// Name implements [fs.DirEntry].
+func (d dirent) Name() string { return path.Base(d.Entry.Name) }
+
+// IsDir implements [fs.DirEntry].
+func (d dirent) IsDir() bool { return d.Entry.Type == typeDir }
+
+// Type implements [fs.DirEntry].
+func (d dirent) Type() fs.FileMode { return fs.FileMode(d.Entry.Mode) & fs.ModeType }
+
+// Info implements [fs.DirEntry].
+func (d dirent) Info() (fs.FileInfo, error) {
+	return &inode{Entry: d.Entry}, nil
 }
 
-func min(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
+// File implements [fs.File] and [fs.ReadDirFile].
+//
+// The ReadDir method errors if called on a non-dir file.
+// The Read methods are implemented by a shared 0-size SectionReader for dir files.
+type file struct {
+	inode
+	*io.SectionReader
+	dirent []fs.DirEntry
+	dirpos int
 }
 
-type dirent struct{ *tar.Header }
+// Interface assertions for file.
+var (
+	_ fs.ReadDirFile = (*file)(nil)
+	_ fs.File        = (*file)(nil)
 
-var _ fs.DirEntry = dirent{}
+	// Extra interfaces that we don't *need* to implement, but do for certain
+	// important use cases (namely reading sqlite databases).
+	_ io.Seeker   = (*file)(nil)
+	_ io.ReaderAt = (*file)(nil)
+)
 
-func (d dirent) Name() string               { return filepath.Base(d.Header.Name) }
-func (d dirent) IsDir() bool                { return d.Header.FileInfo().IsDir() }
-func (d dirent) Type() fs.FileMode          { return d.Header.FileInfo().Mode() & fs.ModeType }
-func (d dirent) Info() (fs.FileInfo, error) { return d.FileInfo(), nil }
+// Close implements [fs.File].
+func (f *file) Close() error { return nil }
 
-// SortDirent returns a function suitable to pass to sort.Slice as a "cmp"
-// function.
-//
-// This is needed because the fs interfaces specify that DirEntry slices
-// returned by the ReadDir methods are sorted lexically.
-func sortDirent(s []fs.DirEntry) func(i, j int) bool {
-	return func(i, j int) bool {
-		return strings.Compare(s[i].Name(), s[j].Name()) == -1
+// Stat implements [fs.File].
+func (f *file) Stat() (fs.FileInfo, error) { return &f.inode, nil }
+
+// ReadDir implements [fs.ReadDirFile].
+func (f *file) ReadDir(n int) ([]fs.DirEntry, error) {
+	if f.Type != `dir` {
+		return nil, &fs.PathError{
+			Op:   `readdir`,
+			Path: f.Entry.Name,
+			Err:  fs.ErrInvalid,
+		}
 	}
+	es := f.dirent[f.dirpos:]
+	end := min(len(es), n)
+	switch {
+	case len(es) == 0 && n <= 0:
+		return nil, nil
+	case len(es) == 0 && n > 0:
+		return nil, io.EOF
+	case n <= 0:
+		end = len(es)
+	default:
+	}
+	f.dirpos += end
+	return es[:end], nil
 }