dockivad: remove, copy and archive trees with 0555 and 000 directories

dockivad/api.go

@@ -118,6 +118,9 @@ func (b *engine) removeImage(ref string, force bool) []obj {
 			continue
 		}
 		p := filepath.Join(b.Cache, entry.Name())
+		if !exists(filepath.Join(p, "image.json")) { // half removed by an earlier engine: goes at the next start (migrateNames)
+			continue
+		}
 		meta := readObject(filepath.Join(p, "image.json"))
 		d := str(meta["digest"])
 		if digest == "" && strings.HasPrefix(d, "sha256:"+strings.TrimPrefix(ref, "sha256:")) {
@@ -150,8 +153,7 @@ func (b *engine) removeImage(ref string, force bool) []obj {
 	}
 	out := []obj{}
 	for p, name := range dirs {
-		retainLowers(b.storeDir(), b.State, p) // containers keep running over its files
-		check(os.RemoveAll(p))
+		removeImageDir(b.storeDir(), b.State, p) // containers keep running over its files
 		out = append(out, obj{"Untagged": name})
 		b.events.publish(imageEvent("untag", digest, digest)) // Docker names both by the image ID
 	}
@@ -344,7 +346,7 @@ func (b *engine) createWith(name string, body obj, prepared *image) obj {
 			if filesDone != nil {
 				<-filesDone
 			}
-			_ = os.RemoveAll(stage)
+			_ = removeTree(stage)
 		}
 	}()
 	allowed := map[string]bool{}
@@ -910,8 +912,7 @@ func (b *engine) api(authorized func(*http.Request) bool) http.Handler {
 			check(e)
 			rows := []any{}
 			for _, entry := range entries {
-				if entry.IsDir() && len(entry.Name()) == 64 {
-					p := filepath.Join(b.Cache, entry.Name(), "image.json")
+				if p := filepath.Join(b.Cache, entry.Name(), "image.json"); entry.IsDir() && len(entry.Name()) == 64 && exists(p) {
 					meta := readObject(p)
 					if !imageFilter(str(meta["name"]), filters) {
 						continue

dockivad/archive.go

@@ -179,7 +179,7 @@ func tarEntry(out *tar.Writer, name, p string, info os.FileInfo) error {
 		return e
 	}
 	if info.Mode().IsRegular() {
-		src, e := os.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+		src, e := openGuest(p)
 		check(e)
 		defer src.Close()
 		_, e = io.CopyN(out, src, h.Size)
@@ -239,7 +239,9 @@ func (b *engine) changes(name string) []obj {
 	}
 	var visit func(rel string) bool
 	visit = func(rel string) bool {
-		entries, e := os.ReadDir(filepath.Join(r.upper, filepath.FromSlash(rel)))
+		dir := filepath.Join(r.upper, filepath.FromSlash(rel))
+		defer unlock(dir, 0o500)()
+		entries, e := os.ReadDir(dir)
 		check(e)
 		changed := false
 		for _, entry := range entries {
@@ -311,7 +313,8 @@ func (b *engine) putArchive(name, requested string, body io.Reader, noOverwriteD
 			if to.files != nil {
 				to.files.remove(to.rel)
 			} else {
-				check(os.RemoveAll(dest))
+				defer unlock(filepath.Dir(dest), 0o700)()
+				check(removeTree(dest))
 			}
 			exists = false
 		}
@@ -323,6 +326,9 @@ func (b *engine) putArchive(name, requested string, body io.Reader, noOverwriteD
 			}
 			defer to.files.settle(to.rel, h.Typeflag == tar.TypeDir)
 		}
+		// Docker's root writes into a directory without write permission: it has it until
+		// the archive is in (restricted.go).
+		defer unlock(filepath.Dir(dest), 0o700)()
 		mode := tarMode(h)
 		switch h.Typeflag {
 		case tar.TypeDir:

dockivad/build.go

@@ -117,7 +117,7 @@ func (b *engine) build(w http.ResponseWriter, r *http.Request, q url.Values) {
 	}()
 	work, e := os.MkdirTemp(private(filepath.Join(b.storeDir(), "build")), "b-")
 	check(e)
-	defer func() { go os.RemoveAll(work) }()
+	defer func() { go removeTree(work) }()
 	quiet := q.Get("q") == "1" || q.Get("q") == "true"
 	var memory int64
 	if raw := q.Get("memory"); raw != "" && raw != "0" { // docker build -m
@@ -707,7 +707,7 @@ func (bd *builder) runStep(st *buildStage, o runOptions) {
 func (bd *builder) discardRoot(st *buildStage) {
 	old := filepath.Join(bd.work, "old-"+token()[:12])
 	check(os.Rename(st.root, old))
-	go os.RemoveAll(old)
+	go removeTree(old)
 }
 
 // The engine writes these into each container; a build keeps the stage's own.
@@ -920,7 +920,7 @@ func isGuestDir(root, guest string) bool {
 // copyIntoGuest copies a directory's contents into a guest directory, merging.
 func copyIntoGuest(src, root, dest string, mode os.FileMode, chmod bool) {
 	guestDir(root, dest)
-	check(filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
+	check(walkTree(src, func(p string, d fs.DirEntry, err error) error {
 		check(err)
 		rel, e := filepath.Rel(src, p)
 		check(e)
@@ -948,6 +948,7 @@ func copyFileTo(src, root, guest string, mode os.FileMode, chmod bool) {
 	parent, base := path.Split(path.Clean("/" + guest))
 	reject(base != "", "Cannot copy a file onto /")
 	dst := filepath.Join(guestDir(root, parent), base)
+	defer unlock(filepath.Dir(dst), 0o700)()
 	if old, e := os.Lstat(dst); e == nil {
 		reject(!old.IsDir(), "Cannot replace the directory "+guest+" with a file")
 		check(os.Remove(dst))

dockivad/buildcache.go

@@ -97,7 +97,7 @@ func (bd *builder) store(st *buildStage, in instruction, id, dir string) {
 	_ = safeRun(func() {
 		tmp, e := os.MkdirTemp(bd.work, "cache-")
 		check(e)
-		defer os.RemoveAll(tmp)
+		defer removeTree(tmp)
 		cloneTree(st.root, filepath.Join(tmp, "rootfs"))
 		now := time.Now().UTC().Format(time.RFC3339Nano)
 		save(filepath.Join(tmp, "config.json"), st.config)
@@ -175,7 +175,7 @@ func (bd *builder) release() {
 // changed, so the sum overstates the disk used. Ask APFS for private sizes if it matters.
 func changedSize(root, base string) int64 {
 	var n int64
-	check(filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+	check(walkTree(root, func(p string, d fs.DirEntry, err error) error {
 		check(err)
 		if !d.Type().IsRegular() {
 			return nil
@@ -185,7 +185,7 @@ func changedSize(root, base string) int64 {
 		if base != "" {
 			rel, e := filepath.Rel(root, p)
 			check(e)
-			if old, e := os.Lstat(filepath.Join(base, rel)); e == nil && old.Mode().IsRegular() && old.Size() == info.Size() && old.ModTime().Equal(info.ModTime()) {
+			if old, e := lstatIn(base, rel); e == nil && old.Mode().IsRegular() && old.Size() == info.Size() && old.ModTime().Equal(info.ModTime()) {
 				return nil
 			}
 		}
@@ -241,7 +241,7 @@ func treeHash(root string) string {
 
 // hashTree writes each entry's name, mode, size and content or symlink target.
 func hashTree(h io.Writer, root string) {
-	check(filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+	check(walkTree(root, func(p string, d fs.DirEntry, err error) error {
 		check(err)
 		rel, e := filepath.Rel(root, p)
 		check(e)
@@ -254,7 +254,7 @@ func hashTree(h io.Writer, root string) {
 			check(e)
 			fmt.Fprintf(h, "%q", target)
 		case info.Mode().IsRegular():
-			f, e := os.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW, 0)
+			f, e := openGuest(p)
 			check(e)
 			defer f.Close()
 			_, e = io.CopyN(h, f, info.Size())
@@ -315,5 +315,5 @@ func (b *engine) discard(dir string) {
 	trash, e := os.MkdirTemp(private(filepath.Join(b.storeDir(), "build")), "trash-")
 	check(e)
 	check(os.Rename(dir, filepath.Join(trash, "entry")))
-	go os.RemoveAll(trash)
+	go removeTree(trash)
 }

dockivad/buildexport.go

@@ -52,7 +52,7 @@ func (w bytesWriter) Write(p []byte) (int, error) {
 func exportTar(c *grpcCall, root string) int64 {
 	counted := &counter{}
 	tw := tar.NewWriter(io.MultiWriter(bytesWriter{c}, counted))
-	check(filepath.WalkDir(root, func(p string, d fs.DirEntry, e error) error {
+	check(walkTree(root, func(p string, d fs.DirEntry, e error) error {
 		if e != nil || p == root {
 			return e
 		}
@@ -78,7 +78,7 @@ func exportTree(c *grpcCall, root string) int64 {
 	listed := make(chan error, 1)
 	go func() {
 		listed <- safeRun(func() {
-			check(filepath.WalkDir(root, func(p string, d fs.DirEntry, e error) error {
+			check(walkTree(root, func(p string, d fs.DirEntry, e error) error {
 				if e != nil || p == root {
 					return e
 				}
@@ -101,7 +101,7 @@ func exportTree(c *grpcCall, root string) int64 {
 				case !mode.IsDir(): // devices, pipes and sockets stay behind
 					return nil
 				}
-				files = append(files, p) // read by the receive loop only after the listing (listed)
+				files = append(files, rel) // read by the receive loop only after the listing (listed)
 				return c.send(pbuf{}.num(1, packetStat).msg(2, st))
 			}))
 			check(c.send(pbuf{}.num(1, packetStat))) // the end of the listing
@@ -132,14 +132,14 @@ func exportTree(c *grpcCall, root string) int64 {
 			waitListing() // the files' numbers are only final once listed
 			id := p.num(3)
 			reject(id < uint64(len(files)), "The client asked for an unknown file")
-			total += sendFile(c, uint32(id), files[id])
+			total += sendFile(c, uint32(id), root, files[id])
 		}
 	}
 }
 
-func sendFile(c *grpcCall, id uint32, p string) int64 {
-	f, e := os.Open(p)
-	check(e)
+func sendFile(c *grpcCall, id uint32, root, rel string) int64 {
+	var f *os.File
+	check(through(root, rel, func(p string) (e error) { f, e = openGuest(p); return })) // restricted.go
 	defer f.Close()
 	buf := make([]byte, exportChunk)
 	var n int64

dockivad/buildkit.go

@@ -25,7 +25,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -627,7 +626,7 @@ func (b *engine) bkControlSolve(ctx context.Context, req pbFields) pbuf {
 			bkSolves.Unlock()
 			st.mu.Lock()
 			if st.llb.work != "" {
-				go os.RemoveAll(st.llb.work)
+				go removeTree(st.llb.work)
 			}
 			st.mu.Unlock()
 		})

dockivad/buildkit_build.go

@@ -36,7 +36,7 @@ func (b *engine) bkBuild(ctx context.Context, st *bkSolve, opts map[string]strin
 	}
 	work, e := os.MkdirTemp(private(filepath.Join(b.storeDir(), "build")), "b-")
 	check(e)
-	defer func() { go os.RemoveAll(work) }()
+	defer func() { go removeTree(work) }()
 	bd := &builder{b: b, ctx: ctx, work: work, context: filepath.Join(work, "context"), network: opts["force-network-mode"],
 		args: map[string]string{}, used: map[string]bool{}, global: platformArgs(platform), platform: platform, nocache: nocache, named: map[string]namedContext{}}
 	bd.secret = func(id string) (value []byte, e error) { // RUN --mount=type=secret: the session's Secrets service

dockivad/buildmount.go

@@ -301,7 +301,7 @@ func (bd *builder) mountSources(mounts []runMount) (binds []any, env []string, t
 	scratch := func() string {
 		d, e := os.MkdirTemp(bd.work, "mount-")
 		check(e)
-		undo = append(undo, func() { os.RemoveAll(d) }) // at once: secrets are among them
+		undo = append(undo, func() { removeTree(d) }) // at once: secrets are among them
 		return d
 	}
 	bind := func(source, target string, readonly bool) {

dockivad/image.go

@@ -401,7 +401,10 @@ func layerDir(root, dir string, create bool) (string, bool) {
 			if !create {
 				return "", false
 			}
-			check(mkdirGuest(p))
+			func() {
+				defer unlock(filepath.Dir(p), 0o700)()
+				check(mkdirGuest(p))
+			}()
 			continue
 		}
 		check(e)
@@ -416,15 +419,27 @@ func layerDir(root, dir string, create bool) (string, bool) {
 // cloneTree copies an unpacked root for a container: one APFS clone of the whole
 // tree (clonefileat), sharing blocks until written; a plain copy elsewhere. A clone
 // turns setuid and setgid off on files and directories (Debian's /var/mail is
-// 2775), so they are set again.
+// 2775), so they are set again. The copy has the modes of src, also where they keep
+// the engine from reading it.
 func cloneTree(src, dst string) {
 	s, e := syscall.BytePtrFromString(src)
 	check(e)
 	d, e := syscall.BytePtrFromString(dst)
 	check(e)
 	cwd := -2 // AT_FDCWD
 	const sysClonefileat, cloneNofollow = 462, 1
-	if _, _, errno := syscall.Syscall6(sysClonefileat, uintptr(cwd), uintptr(unsafe.Pointer(s)), uintptr(cwd), uintptr(unsafe.Pointer(d)), cloneNofollow, 0); errno == 0 {
+	clone := func() syscall.Errno {
+		_, _, errno := syscall.Syscall6(sysClonefileat, uintptr(cwd), uintptr(unsafe.Pointer(s)), uintptr(cwd), uintptr(unsafe.Pointer(d)), cloneNofollow, 0)
+		return errno
+	}
+	errno := clone()
+	var held []locked
+	defer func() { lockTree(dst, held); lockTree(src, held) }()
+	if errno == syscall.EACCES { // a directory or file the guest left unreadable (restricted.go): nothing is copied
+		unlockTree(src, &held)
+		errno = clone()
+	}
+	if errno == 0 {
 		// ponytail: one lstat per entry (~100 ms for 25k files); a list of an image's
 		// setuid files, kept at unpack, would spare it.
 		check(filepath.WalkDir(src, func(p string, entry fs.DirEntry, err error) error {
@@ -582,7 +597,7 @@ func commitImage(cache, name, layout string) string {
 	}
 	stage, e := os.MkdirTemp(cache, ".incoming-")
 	check(e)
-	defer os.RemoveAll(stage)
+	defer removeTree(stage)
 	dest := filepath.Join(stage, "layout")
 	check(os.MkdirAll(filepath.Join(dest, "blobs/sha256"), 0700))
 	for _, d := range im.blobs {
@@ -640,14 +655,20 @@ func (b *engine) commitPulled(name, layout string) string {
 
 // migrateNames moves images cached under a docker.io/ name (docker tag did, before
 // commitImage shortened names) to the familiar name imagePath looks up. One whose
-// familiar name is taken was unreachable anyway, and goes.
+// familiar name is taken was unreachable anyway, and goes. So does a cache directory
+// without its image.json: docker rmi removed that first, then failed on the image's
+// files where a directory lacked write permission, and listing images failed on it.
 func migrateNames(cache string) {
 	entries, e := os.ReadDir(cache)
 	check(e)
 	for _, entry := range entries {
 		_ = safeRun(func() {
 			dir := filepath.Join(cache, entry.Name())
-			if len(entry.Name()) != 64 || !exists(filepath.Join(dir, "image.json")) {
+			if len(entry.Name()) != 64 || !entry.IsDir() {
+				return
+			}
+			if !exists(filepath.Join(dir, "image.json")) { // left by a removal that failed half way
+				removeImageDir(filepath.Dir(cache), filepath.Join(filepath.Dir(cache), "containers"), dir)
 				return
 			}
 			meta := readObject(filepath.Join(dir, "image.json"))
@@ -657,8 +678,7 @@ func migrateNames(cache string) {
 			}
 			target := filepath.Join(cache, key(shortName(name)))
 			if exists(target) {
-				retainLowers(filepath.Dir(cache), filepath.Join(filepath.Dir(cache), "containers"), dir)
-				check(os.RemoveAll(dir))
+				removeImageDir(filepath.Dir(cache), filepath.Join(filepath.Dir(cache), "containers"), dir)
 				return
 			}
 			meta["name"] = shortName(name)
@@ -676,7 +696,7 @@ func ensureUnpacked(target string) {
 	im := loadImage(noLinks(target, "layout"))
 	stage, e := os.MkdirTemp(target, ".rootfs-")
 	check(e)
-	defer os.RemoveAll(stage)
+	defer removeTree(stage)
 	size := im.unpack(stage)
 	if e := os.Rename(stage, filepath.Join(target, "rootfs")); e != nil && !exists(filepath.Join(target, "rootfs")) {
 		check(e)