From 491cd0c6df5dd77826fc320c3eb855947f1cd056 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 26 Nov 2025 11:01:08 +0100 Subject: [PATCH 01/65] feat: search, load and validates manifests Signed-off-by: Paul Mars --- cmd/chisel/cmd_cut.go | 16 +++ internal/manifestutil/manifestutil.go | 45 ++++++- internal/slicer/check.go | 184 ++++++++++++++++++++++++++ internal/slicer/slicer.go | 92 +++++++++++++ public/manifest/manifest.go | 4 + 5 files changed, 334 insertions(+), 7 deletions(-) create mode 100644 internal/slicer/check.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79..cca9055f 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) var shortCutHelp = "Cut a tree with selected slices" @@ -73,6 +74,21 @@ func (cmd *cmdCut) Execute(args []string) error { } } + mfest, err := slicer.Inspect(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) + } + selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b05402..f6bb7e60 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -18,22 +18,44 @@ import ( const DefaultFilename = "manifest.wall" +func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + collector(path, slice) + } + } +} + // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { manifestSlices := make(map[string][]*setup.Slice) + collector := func(path string, slice *setup.Slice) { + manifestSlices[path] = append(manifestSlices[path], slice) + } for _, slice := range slices { - for path, info := range slice.Contents { - if info.Generate == setup.GenerateManifest { - dir := strings.TrimSuffix(path, "**") - path = filepath.Join(dir, DefaultFilename) - manifestSlices[path] = append(manifestSlices[path], slice) - } - } + collectManifests(slice, collector) } return manifestSlices } +// FindPathsInRelease finds all the paths marked with "generate:manifest" +// for the given release. +func FindPathsInRelease(r *setup.Release) []string { + manifestPaths := make([]string,0) + collector := func(path string, slice *setup.Slice) { + manifestPaths = append(manifestPaths, path) + } + for _, pkg := range r.Packages { + for _, slice := range pkg.Slices { + collectManifests(slice, collector) + } + } + return manifestPaths +} + type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice @@ -340,3 +362,12 @@ func Validate(mfest *manifest.Manifest) (err error) { } return nil } + +// CompareSchemas compares two manifest schema strings. +func CompareSchemas(va, vb string) int { + if va == manifest.Schema && va == vb { + return 0 + } + return -1 +} + diff --git a/internal/slicer/check.go b/internal/slicer/check.go new file mode 100644 index 00000000..ccf97605 --- /dev/null +++ b/internal/slicer/check.go @@ -0,0 +1,184 @@ +package slicer + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" + + "github.com/klauspost/compress/zstd" + + "github.com/canonical/chisel/internal/manifestutil" + "github.com/canonical/chisel/public/manifest" +) + +type pathInfo struct { + mode string + size int64 + link string + hash string +} + +func unixPerm(mode fs.FileMode) (perm uint32) { + perm = uint32(mode.Perm()) + if mode&fs.ModeSticky != 0 { + perm |= 0o1000 + } + return perm +} + +// checkRootDir checks the content of the target directory matches with +// the manifest. Files not managed by chisel are ignored. +// This function works under the assumption the manifest is valid. +func checkRootDir(mfest *manifest.Manifest, rootDir string) error { + singlePathsByFSInode := make(map[uint64]string) + fsInodeByManifestInode := make(map[uint64]uint64) + manifestInfos := make(map[string]*pathInfo) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + pathHash := path.FinalSHA256 + if pathHash == "" { + pathHash = path.SHA256 + } + recordedPathInfo := &pathInfo{ + mode: path.Mode, + size: int64(path.Size), + link: path.Link, + hash: pathHash, + } + + fsInfo := &pathInfo{} + fullPath := filepath.Join(rootDir, path.Path) + info, err := os.Lstat(fullPath) + if err != nil { + return err + } + mode := info.Mode() + fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode)) + ftype := mode & fs.ModeType + switch ftype { + case fs.ModeDir: + // Nothing to do. + case fs.ModeSymlink: + fsInfo.link, err = os.Readlink(fullPath) + if err != nil { + return fmt.Errorf("cannot read symlink %q: %w", fullPath, err) + } + case 0: // Regular file. + h, err := contentHash(fullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err) + } + fsInfo.hash = hex.EncodeToString(h) + fsInfo.size = info.Size() + default: + return fmt.Errorf("cannot check %q: unrecognized type %s", fullPath, mode.String()) + } + + // Collect manifests for tailored checking later. Adjust observed hash and + // size to still compare in a generic way. + if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" { + mfestInfo := *fsInfo + manifestInfos[path.Path] = &mfestInfo + fsInfo.size = 0 + fsInfo.hash = "" + } + + if recordedPathInfo.mode != fsInfo.mode { + return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode) + } + if recordedPathInfo.size != fsInfo.size { + return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size) + } + if recordedPathInfo.link != fsInfo.link { + return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link) + } + if recordedPathInfo.hash != fsInfo.hash { + return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash) + } + // Check hardlink. + if ftype != fs.ModeDir { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("cannot get syscall stat info for %q", info.Name()) + } + inode := stat.Ino + + if path.Inode == 0 { + // This path must not be linked to any other. + singlePath, ok := singlePathsByFSInode[inode] + if ok { + return fmt.Errorf("inconsistent content at %q: recorded no hardlink, observed hardlinked to %q", path.Path, singlePath) + } + singlePathsByFSInode[inode] = path.Path + } else { + recordedInode, ok := fsInodeByManifestInode[path.Inode] + if !ok { + fsInodeByManifestInode[path.Inode] = inode + } else if recordedInode != inode { + return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path) + } + } + } + return nil + }) + if err != nil { + return err + } + + // Check manifests. + // They must all be valid manifests and be consistent per schema version. + schemaManifestInfos := make(map[string]*pathInfo) + for path, info := range manifestInfos { + fullPath := filepath.Join(rootDir, path) + f, err := os.Open(fullPath) + if err != nil { + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err = manifest.Read(r) + if err != nil { + return err + } + err = manifestutil.Validate(mfest) + if err != nil { + return err + } + schema := mfest.Schema() + refInfo, ok := schemaManifestInfos[schema] + if !ok { + schemaManifestInfos[schema] = info + continue + } + + if refInfo.size != info.size { + return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size) + } + if refInfo.hash != info.hash { + return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash) + } + } + return nil +} + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..48eb0e11 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,12 @@ package slicer import ( "archive/tar" "bytes" + "encoding/hex" "fmt" "io" "io/fs" "os" + "path" "path/filepath" "slices" "sort" @@ -21,6 +23,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) const manifestMode fs.FileMode = 0644 @@ -537,3 +540,92 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel } return pkgArchive, nil } + +// Inspect examines and validates the targetDir. It returns, if found and valid +// the manifest representing the content in the targetDir. +func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) { + var mfest *manifest.Manifest + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) > 0 { + logf("Inspecting root directory...") + var err error + mfest, err = selectValidManifest(targetDir, manifestPaths) + if err != nil { + return nil, err + } + if mfest != nil { + err = checkRootDir(mfest, targetDir) + if err != nil { + return nil, err + } + } + } + return mfest, nil +} + +// selectValidManifest returns, if found, a valid manifest with the latest +// schema. Consistency with all other manifests with the same schema is verified +// so the selection is deterministic. +func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) { + targetDir = filepath.Clean(targetDir) + if !filepath.IsAbs(targetDir) { + dir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("cannot obtain current directory: %w", err) + } + targetDir = filepath.Join(dir, targetDir) + } + + type manifestHash struct { + path string + hash string + } + var selected *manifest.Manifest + schemaManifest := make(map[string]manifestHash) + for _, mfestPath := range manifestPaths { + err := func() error { + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + return nil + } + err = manifestutil.Validate(mfest) + if err != nil { + return nil + } + + if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) + } + selected = mfest + } + return nil + }() + if err != nil { + return nil, err + } + } + return selected, nil +} diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..65b362d5 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -68,6 +68,10 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return manifest, nil } +func (manifest *Manifest) Schema() string { + return manifest.db.Schema() +} + func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) } From a6e57f7290104bec2c17bfefc508d091badc971d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 29 Jan 2026 17:19:12 +0100 Subject: [PATCH 02/65] feat: adapt to revised strategy --- cmd/chisel/cmd_cut.go | 3 +- internal/fsutil/create.go | 16 +- internal/manifestutil/manifestutil.go | 3 +- internal/manifestutil/manifestutil_test.go | 1 - internal/slicer/check.go | 184 --------------------- internal/slicer/slicer.go | 123 +++++++++----- 6 files changed, 100 insertions(+), 230 deletions(-) delete mode 100644 internal/slicer/check.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index cca9055f..940ec121 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -74,7 +74,7 @@ func (cmd *cmdCut) Execute(args []string) error { } } - mfest, err := slicer.Inspect(cmd.RootDir, release) + mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) if err != nil { return err } @@ -141,6 +141,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + Manifest: mfest, }) return err } diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 0503c96f..2f87f670 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -166,11 +166,19 @@ func createDir(o *CreateOptions) error { if err != nil { return err } - err = os.Mkdir(path, o.Mode) - if os.IsExist(err) { - return nil + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err } - return err + return os.Mkdir(path, o.Mode) } func createFile(o *CreateOptions) error { diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index f6bb7e60..b2946d57 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -44,7 +44,7 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { // FindPathsInRelease finds all the paths marked with "generate:manifest" // for the given release. func FindPathsInRelease(r *setup.Release) []string { - manifestPaths := make([]string,0) + manifestPaths := make([]string, 0) collector := func(path string, slice *setup.Slice) { manifestPaths = append(manifestPaths, path) } @@ -370,4 +370,3 @@ func CompareSchemas(va, vb string) int { } return -1 } - diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 2bab0a68..0c6c0c06 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -2,7 +2,6 @@ package manifestutil_test import ( "bytes" - "io" "io/fs" "os" "path" diff --git a/internal/slicer/check.go b/internal/slicer/check.go deleted file mode 100644 index ccf97605..00000000 --- a/internal/slicer/check.go +++ /dev/null @@ -1,184 +0,0 @@ -package slicer - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "syscall" - - "github.com/klauspost/compress/zstd" - - "github.com/canonical/chisel/internal/manifestutil" - "github.com/canonical/chisel/public/manifest" -) - -type pathInfo struct { - mode string - size int64 - link string - hash string -} - -func unixPerm(mode fs.FileMode) (perm uint32) { - perm = uint32(mode.Perm()) - if mode&fs.ModeSticky != 0 { - perm |= 0o1000 - } - return perm -} - -// checkRootDir checks the content of the target directory matches with -// the manifest. Files not managed by chisel are ignored. -// This function works under the assumption the manifest is valid. -func checkRootDir(mfest *manifest.Manifest, rootDir string) error { - singlePathsByFSInode := make(map[uint64]string) - fsInodeByManifestInode := make(map[uint64]uint64) - manifestInfos := make(map[string]*pathInfo) - err := mfest.IteratePaths("", func(path *manifest.Path) error { - pathHash := path.FinalSHA256 - if pathHash == "" { - pathHash = path.SHA256 - } - recordedPathInfo := &pathInfo{ - mode: path.Mode, - size: int64(path.Size), - link: path.Link, - hash: pathHash, - } - - fsInfo := &pathInfo{} - fullPath := filepath.Join(rootDir, path.Path) - info, err := os.Lstat(fullPath) - if err != nil { - return err - } - mode := info.Mode() - fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode)) - ftype := mode & fs.ModeType - switch ftype { - case fs.ModeDir: - // Nothing to do. - case fs.ModeSymlink: - fsInfo.link, err = os.Readlink(fullPath) - if err != nil { - return fmt.Errorf("cannot read symlink %q: %w", fullPath, err) - } - case 0: // Regular file. - h, err := contentHash(fullPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err) - } - fsInfo.hash = hex.EncodeToString(h) - fsInfo.size = info.Size() - default: - return fmt.Errorf("cannot check %q: unrecognized type %s", fullPath, mode.String()) - } - - // Collect manifests for tailored checking later. Adjust observed hash and - // size to still compare in a generic way. - if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" { - mfestInfo := *fsInfo - manifestInfos[path.Path] = &mfestInfo - fsInfo.size = 0 - fsInfo.hash = "" - } - - if recordedPathInfo.mode != fsInfo.mode { - return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode) - } - if recordedPathInfo.size != fsInfo.size { - return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size) - } - if recordedPathInfo.link != fsInfo.link { - return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link) - } - if recordedPathInfo.hash != fsInfo.hash { - return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash) - } - // Check hardlink. - if ftype != fs.ModeDir { - stat, ok := info.Sys().(*syscall.Stat_t) - if !ok { - return fmt.Errorf("cannot get syscall stat info for %q", info.Name()) - } - inode := stat.Ino - - if path.Inode == 0 { - // This path must not be linked to any other. - singlePath, ok := singlePathsByFSInode[inode] - if ok { - return fmt.Errorf("inconsistent content at %q: recorded no hardlink, observed hardlinked to %q", path.Path, singlePath) - } - singlePathsByFSInode[inode] = path.Path - } else { - recordedInode, ok := fsInodeByManifestInode[path.Inode] - if !ok { - fsInodeByManifestInode[path.Inode] = inode - } else if recordedInode != inode { - return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path) - } - } - } - return nil - }) - if err != nil { - return err - } - - // Check manifests. - // They must all be valid manifests and be consistent per schema version. - schemaManifestInfos := make(map[string]*pathInfo) - for path, info := range manifestInfos { - fullPath := filepath.Join(rootDir, path) - f, err := os.Open(fullPath) - if err != nil { - return err - } - defer f.Close() - r, err := zstd.NewReader(f) - if err != nil { - return err - } - defer r.Close() - mfest, err = manifest.Read(r) - if err != nil { - return err - } - err = manifestutil.Validate(mfest) - if err != nil { - return err - } - schema := mfest.Schema() - refInfo, ok := schemaManifestInfos[schema] - if !ok { - schemaManifestInfos[schema] = info - continue - } - - if refInfo.size != info.size { - return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size) - } - if refInfo.hash != info.hash { - return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash) - } - } - return nil -} - -func contentHash(path string) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { - return nil, err - } - return h.Sum(nil), nil -} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 48eb0e11..1dcfd019 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,12 @@ package slicer import ( "archive/tar" "bytes" + "crypto/sha256" "encoding/hex" "fmt" "io" "io/fs" + "maps" "os" "path" "path/filepath" @@ -32,6 +34,7 @@ type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + Manifest *manifest.Manifest } type pathData struct { @@ -93,6 +96,19 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } + var originalTargetDir string + if options.Manifest != nil { + tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") + if err != nil { + return fmt.Errorf("cannot create temporary working directory: %w", err) + } + originalTargetDir = targetDir + targetDir = tmpWorkDir + defer func() { + os.RemoveAll(tmpWorkDir) + }() + } + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { return err @@ -350,7 +366,41 @@ func Run(options *RunOptions) error { return err } - return generateManifests(targetDir, options.Selection, report, pkgInfos) + err = generateManifests(targetDir, options.Selection, report, pkgInfos) + if err != nil { + return err + } + + if options.Manifest != nil { + return upgrade(originalTargetDir, targetDir, report, options.Manifest) + } + return nil +} + +// upgrade upgrades content in targetDir using content in tempDir. +// Work on sorted list of content in tempDir +func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { + paths := slices.Sorted(maps.Keys(newReport.Entries)) + for _, path := range paths { + entry := newReport.Entries[path] + var err error + switch entry.Mode & fs.ModeType { + case 0: + // rename file if hash different than same path in old manifest + case fs.ModeDir: + // create dir with proper mode + // or chmod existing dir + case fs.ModeSymlink: + // move symlink to dest + default: + err = fmt.Errorf("unsupported file type: %s", path) + } + if err != nil { + return err + } + } + + return nil } func generateManifests(targetDir string, selection *setup.Selection, @@ -541,32 +591,10 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return pkgArchive, nil } -// Inspect examines and validates the targetDir. It returns, if found and valid -// the manifest representing the content in the targetDir. -func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) { - var mfest *manifest.Manifest - manifestPaths := manifestutil.FindPathsInRelease(release) - if len(manifestPaths) > 0 { - logf("Inspecting root directory...") - var err error - mfest, err = selectValidManifest(targetDir, manifestPaths) - if err != nil { - return nil, err - } - if mfest != nil { - err = checkRootDir(mfest, targetDir) - if err != nil { - return nil, err - } - } - } - return mfest, nil -} - -// selectValidManifest returns, if found, a valid manifest with the latest +// SelectValidManifest returns, if found, a valid manifest with the latest // schema. Consistency with all other manifests with the same schema is verified // so the selection is deterministic. -func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) { +func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { targetDir = filepath.Clean(targetDir) if !filepath.IsAbs(targetDir) { dir, err := os.Getwd() @@ -575,6 +603,10 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma } targetDir = filepath.Join(dir, targetDir) } + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) == 0 { + return nil, nil + } type manifestHash struct { path string @@ -600,25 +632,26 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma defer r.Close() mfest, err := manifest.Read(r) if err != nil { - return nil + return err } err = manifestutil.Validate(mfest) if err != nil { - return nil + return err + } + // Verify consistency with other manifests with the same schema. + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) } if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { - h, err := contentHash(mfestFullPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) - } - mfestHash := hex.EncodeToString(h) - refMfest, ok := schemaManifest[mfest.Schema()] - if !ok { - schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} - } else if refMfest.hash != mfestHash { - return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) - } selected = mfest } return nil @@ -629,3 +662,17 @@ func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Ma } return selected, nil } + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} From 5da04501bd541faf4a898c2ca8a0b8c564210ac8 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:26:15 +0100 Subject: [PATCH 03/65] feat: implement upgrade --- internal/fsutil/create.go | 3 ++ internal/slicer/slicer.go | 99 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 2f87f670..60bfc2b0 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -169,6 +169,9 @@ func createDir(o *CreateOptions) error { fileinfo, err := os.Lstat(path) if err == nil { if fileinfo.IsDir() { + if fileinfo.Mode() != o.Mode && o.OverrideMode { + return os.Chmod(path, o.Mode) + } return nil } err = os.Remove(path) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 1dcfd019..67949d10 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -377,21 +377,56 @@ func Run(options *RunOptions) error { return nil } +// absPath requires root to be a clean path that ends in "/". +func absPath(root, relPath string) (string, error) { + path := filepath.Clean(filepath.Join(root, relPath)) + if !strings.HasPrefix(path, root) { + return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) + } + return path, nil +} + // upgrade upgrades content in targetDir using content in tempDir. -// Work on sorted list of content in tempDir func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { + logf("Upgrading existing content...") + filesToDelete := make([]*manifest.Path, 0) + oldPaths := make(map[string]*manifest.Path, 0) + err := oldManifest.IteratePaths("", func(path *manifest.Path) error { + _, ok := newReport.Entries[path.Path] + if !ok && strings.HasSuffix(path.Path, "/") { + // Keep directories. + filesToDelete = append(filesToDelete, path) + return nil + } + oldPaths[path.Path] = path + return nil + }) + if err != nil { + return err + } + paths := slices.Sorted(maps.Keys(newReport.Entries)) for _, path := range paths { + srcPath, err := absPath(tempDir, path) + if err != nil { + return err + } + dstPath, err := absPath(targetDir, path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + entry := newReport.Entries[path] - var err error switch entry.Mode & fs.ModeType { case 0: - // rename file if hash different than same path in old manifest + err = upgradeFile(srcPath, dstPath, &entry) case fs.ModeDir: - // create dir with proper mode - // or chmod existing dir + err = upgradeDir(dstPath, &entry) case fs.ModeSymlink: - // move symlink to dest + err = os.Rename(srcPath, dstPath) default: err = fmt.Errorf("unsupported file type: %s", path) } @@ -400,9 +435,61 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } + // Delete old files + for _, pathToDelete := range filesToDelete { + path, err := absPath(tempDir, pathToDelete.Path) + if err != nil { + return err + } + err = os.Remove(path) + if err != nil { + return err + } + } return nil } +func upgradeFile(srcPath string, dstPath string, entry *manifestutil.ReportEntry) error { + fileinfo, err := os.Lstat(dstPath) + if err == nil { + h, err := contentHash(dstPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", dstPath, err) + } + oldHash := hex.EncodeToString(h) + newHash := entry.SHA256 + if newHash == "" { + newHash = entry.FinalSHA256 + } + if oldHash == newHash && entry.Mode == fileinfo.Mode() { + // Same file, do nothing. + return nil + } + } else if !os.IsNotExist(err) { + return err + } + return os.Rename(srcPath, dstPath) +} + +func upgradeDir(path string, entry *manifestutil.ReportEntry) error { + fileinfo, err := os.Lstat(path) + if err == nil { + if fileinfo.IsDir() { + if fileinfo.Mode() != entry.Mode { + return os.Chmod(path, entry.Mode) + } + return nil + } + err = os.Remove(path) + if err != nil { + return err + } + } else if !os.IsNotExist(err) { + return err + } + return os.Mkdir(path, entry.Mode) +} + func generateManifests(targetDir string, selection *setup.Selection, report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { manifestSlices := manifestutil.FindPaths(selection.Slices) From 5703751eff9926771d086ee7d43480e916848522 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:43:30 +0100 Subject: [PATCH 04/65] fix: deletion --- cmd/chisel/cmd_cut.go | 2 +- internal/slicer/slicer.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 940ec121..5873c3a3 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -141,7 +141,7 @@ func (cmd *cmdCut) Execute(args []string) error { Selection: selection, Archives: archives, TargetDir: cmd.RootDir, - Manifest: mfest, + Manifest: mfest, }) return err } diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 67949d10..aed6f668 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -393,7 +393,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o oldPaths := make(map[string]*manifest.Path, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] - if !ok && strings.HasSuffix(path.Path, "/") { + if !ok && !strings.HasSuffix(path.Path, "/") { // Keep directories. filesToDelete = append(filesToDelete, path) return nil @@ -437,7 +437,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o // Delete old files for _, pathToDelete := range filesToDelete { - path, err := absPath(tempDir, pathToDelete.Path) + path, err := absPath(targetDir, pathToDelete.Path) if err != nil { return err } From 26bb2b3361c8fbdb078438eeeb9e144815e6da67 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 14:49:19 +0100 Subject: [PATCH 05/65] fix: revert inadvertent change --- internal/manifestutil/manifestutil_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 0c6c0c06..2bab0a68 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -2,6 +2,7 @@ package manifestutil_test import ( "bytes" + "io" "io/fs" "os" "path" From 4a14230ed08272abf1a7e4badeca243f6bcae7cc Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 15:08:17 +0100 Subject: [PATCH 06/65] fix: check slice collecting error --- cmd/chisel/cmd_cut.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 5873c3a3..b0dd1eec 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -79,7 +79,7 @@ func (cmd *cmdCut) Execute(args []string) error { return err } if mfest != nil { - mfest.IterateSlices("", func(slice *manifest.Slice) error { + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { sk, err := setup.ParseSliceKey(slice.Name) if err != nil { return err @@ -87,6 +87,9 @@ func (cmd *cmdCut) Execute(args []string) error { sliceKeys = append(sliceKeys, sk) return nil }) + if err != nil { + return err + } } selection, err := setup.Select(release, sliceKeys, cmd.Arch) From 865c1b8ae17a03e70df70b2ca7505e5de1053b15 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 17:01:00 +0100 Subject: [PATCH 07/65] fix: simplify upgrade and improve deletion --- internal/slicer/slicer.go | 55 ++++++++++++++------------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index aed6f668..5de5813d 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -389,13 +389,12 @@ func absPath(root, relPath string) (string, error) { // upgrade upgrades content in targetDir using content in tempDir. func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { logf("Upgrading existing content...") - filesToDelete := make([]*manifest.Path, 0) + pathsToDelete := make([]string, 0) oldPaths := make(map[string]*manifest.Path, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] - if !ok && !strings.HasSuffix(path.Path, "/") { - // Keep directories. - filesToDelete = append(filesToDelete, path) + if !ok { + pathsToDelete = append(pathsToDelete, path.Path) return nil } oldPaths[path.Path] = path @@ -422,11 +421,10 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o entry := newReport.Entries[path] switch entry.Mode & fs.ModeType { case 0: - err = upgradeFile(srcPath, dstPath, &entry) - case fs.ModeDir: - err = upgradeDir(dstPath, &entry) case fs.ModeSymlink: err = os.Rename(srcPath, dstPath) + case fs.ModeDir: + err = upgradeDir(dstPath, &entry) default: err = fmt.Errorf("unsupported file type: %s", path) } @@ -435,42 +433,29 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } - // Delete old files - for _, pathToDelete := range filesToDelete { - path, err := absPath(targetDir, pathToDelete.Path) + // Delete old paths + slices.Sort(pathsToDelete) + slices.Reverse(pathsToDelete) + for _, pathToDelete := range pathsToDelete { + path, err := absPath(targetDir, pathToDelete) if err != nil { return err } - err = os.Remove(path) - if err != nil { - return err + if strings.HasSuffix(path, "/") { + err = syscall.Rmdir(path) + if err != nil && err != syscall.ENOTEMPTY { + return err + } + } else { + err = os.Remove(path) + if err != nil { + return err + } } } return nil } -func upgradeFile(srcPath string, dstPath string, entry *manifestutil.ReportEntry) error { - fileinfo, err := os.Lstat(dstPath) - if err == nil { - h, err := contentHash(dstPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", dstPath, err) - } - oldHash := hex.EncodeToString(h) - newHash := entry.SHA256 - if newHash == "" { - newHash = entry.FinalSHA256 - } - if oldHash == newHash && entry.Mode == fileinfo.Mode() { - // Same file, do nothing. - return nil - } - } else if !os.IsNotExist(err) { - return err - } - return os.Rename(srcPath, dstPath) -} - func upgradeDir(path string, entry *manifestutil.ReportEntry) error { fileinfo, err := os.Lstat(path) if err == nil { From 0343ddcfe0db4d58613e7223ea6adb726ee2e559 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 30 Jan 2026 17:12:33 +0100 Subject: [PATCH 08/65] refactor: cleaning --- internal/slicer/slicer.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 5de5813d..7923d6cc 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -389,15 +389,12 @@ func absPath(root, relPath string) (string, error) { // upgrade upgrades content in targetDir using content in tempDir. func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { logf("Upgrading existing content...") - pathsToDelete := make([]string, 0) - oldPaths := make(map[string]*manifest.Path, 0) + missingPaths := make([]string, 0) err := oldManifest.IteratePaths("", func(path *manifest.Path) error { _, ok := newReport.Entries[path.Path] if !ok { - pathsToDelete = append(pathsToDelete, path.Path) - return nil + missingPaths = append(missingPaths, path.Path) } - oldPaths[path.Path] = path return nil }) if err != nil { @@ -433,11 +430,11 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } } - // Delete old paths - slices.Sort(pathsToDelete) - slices.Reverse(pathsToDelete) - for _, pathToDelete := range pathsToDelete { - path, err := absPath(targetDir, pathToDelete) + // Remove missing paths + slices.Sort(missingPaths) + slices.Reverse(missingPaths) + for _, relPath := range missingPaths { + path, err := absPath(targetDir, relPath) if err != nil { return err } From e11642485b9ffea7ee3bc008bfbaa43e40c4d0d1 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 2 Feb 2026 11:59:10 +0100 Subject: [PATCH 09/65] refactor: refine upgrade --- internal/slicer/slicer.go | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 7923d6cc..2e263fd5 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -386,22 +386,10 @@ func absPath(root, relPath string) (string, error) { return path, nil } -// upgrade upgrades content in targetDir using content in tempDir. -func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, oldManifest *manifest.Manifest) error { - logf("Upgrading existing content...") - missingPaths := make([]string, 0) - err := oldManifest.IteratePaths("", func(path *manifest.Path) error { - _, ok := newReport.Entries[path.Path] - if !ok { - missingPaths = append(missingPaths, path.Path) - } - return nil - }) - if err != nil { - return err - } - - paths := slices.Sorted(maps.Keys(newReport.Entries)) +// upgrade upgrades content in targetDir with content in tempDir. +func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfest *manifest.Manifest) error { + logf("Upgrading content...") + paths := slices.Sorted(maps.Keys(report.Entries)) for _, path := range paths { srcPath, err := absPath(tempDir, path) if err != nil { @@ -415,7 +403,7 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o return err } - entry := newReport.Entries[path] + entry := report.Entries[path] switch entry.Mode & fs.ModeType { case 0: case fs.ModeSymlink: @@ -431,8 +419,18 @@ func upgrade(targetDir string, tempDir string, newReport *manifestutil.Report, o } // Remove missing paths - slices.Sort(missingPaths) - slices.Reverse(missingPaths) + missingPaths := make([]string, 0) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + _, ok := report.Entries[path.Path] + if !ok { + missingPaths = append(missingPaths, path.Path) + } + return nil + }) + if err != nil { + return err + } + sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) for _, relPath := range missingPaths { path, err := absPath(targetDir, relPath) if err != nil { From 5b9d408ba4f1c75719b48900919c2d79e508ebfe Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 09:31:08 +0100 Subject: [PATCH 10/65] refactor: fsutil handles moving and removing --- internal/fsutil/move.go | 86 +++++++++++++++++++++++++++++++++++++++ internal/fsutil/remove.go | 50 +++++++++++++++++++++++ internal/slicer/slicer.go | 73 ++++++--------------------------- 3 files changed, 148 insertions(+), 61 deletions(-) create mode 100644 internal/fsutil/move.go create mode 100644 internal/fsutil/remove.go diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go new file mode 100644 index 00000000..4276d8a6 --- /dev/null +++ b/internal/fsutil/move.go @@ -0,0 +1,86 @@ +package fsutil + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" +) + +type MoveOptions struct { + SrcRoot string + DstRoot string + // Path is relative to Root. + Path string + Mode fs.FileMode + // If MakeParents is true, missing parent directories of Path are + // created with permissions 0755. + MakeParents bool + // If OverrideMode is true and entry already exists, update the mode. Does + // not affect symlinks. + OverrideMode bool +} + +// Move moves a filesystem entry according to the provided options. +// +// Move can return errors from the os package. +func Move(options *MoveOptions) error { + o, err := getValidMoveOptions(options) + if err != nil { + return err + } + + srcPath, err := absPath(options.SrcRoot, o.Path) + if err != nil { + return err + } + dstPath, err := absPath(options.DstRoot, o.Path) + if err != nil { + return err + } + + if o.MakeParents { + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + } + + switch o.Mode & fs.ModeType { + case 0: + case fs.ModeSymlink: + err = os.Rename(srcPath, dstPath) + case fs.ModeDir: + createOptions := &CreateOptions{ + Root: o.DstRoot, + Path: o.Path, + Mode: o.Mode, + OverrideMode: o.OverrideMode, + } + err = createDir(createOptions) + default: + err = fmt.Errorf("unsupported file type: %s", o.Path) + } + if err != nil { + return err + } + + return nil +} + +func getValidMoveOptions(options *MoveOptions) (*MoveOptions, error) { + optsCopy := *options + o := &optsCopy + if o.SrcRoot == "" { + return nil, fmt.Errorf("internal error: MoveOptions.SrcRoot is unset") + } + if o.DstRoot == "" { + return nil, fmt.Errorf("internal error: MoveOptions.DstRoot is unset") + } + if o.SrcRoot != "/" { + o.SrcRoot = filepath.Clean(o.SrcRoot) + "/" + } + if o.DstRoot != "/" { + o.DstRoot = filepath.Clean(o.DstRoot) + "/" + } + return o, nil +} diff --git a/internal/fsutil/remove.go b/internal/fsutil/remove.go new file mode 100644 index 00000000..6a2c731c --- /dev/null +++ b/internal/fsutil/remove.go @@ -0,0 +1,50 @@ +package fsutil + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "syscall" +) + +type RemoveOptions struct { + Root string + // Path is relative to Root. + Path string +} + +func Remove(options *RemoveOptions) error { + options, err := getValidRemoveOptions(options) + if err != nil { + return err + } + path, err := absPath(options.Root, options.Path) + if err != nil { + return err + } + if strings.HasSuffix(path, "/") { + err = syscall.Rmdir(path) + if err != nil && err != syscall.ENOTEMPTY { + return err + } + } else { + err = os.Remove(path) + if err != nil { + return err + } + } + return nil +} + +func getValidRemoveOptions(options *RemoveOptions) (*RemoveOptions, error) { + optsCopy := *options + o := &optsCopy + if o.Root == "" { + return nil, fmt.Errorf("internal error: RemoveOptions.Root is unset") + } + if o.Root != "/" { + o.Root = filepath.Clean(o.Root) + "/" + } + return o, nil +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2e263fd5..2ccab61b 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -377,42 +377,20 @@ func Run(options *RunOptions) error { return nil } -// absPath requires root to be a clean path that ends in "/". -func absPath(root, relPath string) (string, error) { - path := filepath.Clean(filepath.Join(root, relPath)) - if !strings.HasPrefix(path, root) { - return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) - } - return path, nil -} - // upgrade upgrades content in targetDir with content in tempDir. func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfest *manifest.Manifest) error { logf("Upgrading content...") paths := slices.Sorted(maps.Keys(report.Entries)) for _, path := range paths { - srcPath, err := absPath(tempDir, path) - if err != nil { - return err - } - dstPath, err := absPath(targetDir, path) - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { - return err - } - entry := report.Entries[path] - switch entry.Mode & fs.ModeType { - case 0: - case fs.ModeSymlink: - err = os.Rename(srcPath, dstPath) - case fs.ModeDir: - err = upgradeDir(dstPath, &entry) - default: - err = fmt.Errorf("unsupported file type: %s", path) - } + err := fsutil.Move(&fsutil.MoveOptions{ + SrcRoot: tempDir, + DstRoot: targetDir, + Path: path, + Mode: entry.Mode, + MakeParents: true, + OverrideMode: true, + }) if err != nil { return err } @@ -432,44 +410,17 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes } sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) for _, relPath := range missingPaths { - path, err := absPath(targetDir, relPath) + err := fsutil.Remove(&fsutil.RemoveOptions{ + Root: targetDir, + Path: relPath, + }) if err != nil { return err } - if strings.HasSuffix(path, "/") { - err = syscall.Rmdir(path) - if err != nil && err != syscall.ENOTEMPTY { - return err - } - } else { - err = os.Remove(path) - if err != nil { - return err - } - } } return nil } -func upgradeDir(path string, entry *manifestutil.ReportEntry) error { - fileinfo, err := os.Lstat(path) - if err == nil { - if fileinfo.IsDir() { - if fileinfo.Mode() != entry.Mode { - return os.Chmod(path, entry.Mode) - } - return nil - } - err = os.Remove(path) - if err != nil { - return err - } - } else if !os.IsNotExist(err) { - return err - } - return os.Mkdir(path, entry.Mode) -} - func generateManifests(targetDir string, selection *setup.Selection, report *manifestutil.Report, pkgInfos []*archive.PackageInfo) error { manifestSlices := manifestutil.FindPaths(selection.Slices) From 7861c8dfbfa958310fd883b16de2a2c66d508522 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 09:40:38 +0100 Subject: [PATCH 11/65] refactor: improve consistency --- internal/fsutil/create.go | 6 +++--- internal/fsutil/move.go | 6 +++--- internal/fsutil/remove.go | 4 ++++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 60bfc2b0..e4987e64 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -42,7 +42,7 @@ type Entry struct { // // Create can return errors from the os package. func Create(options *CreateOptions) (*Entry, error) { - o, err := getValidOptions(options) + o, err := getValidCreateOptions(options) if err != nil { return nil, err } @@ -124,7 +124,7 @@ func Create(options *CreateOptions) (*Entry, error) { // information recorded in Entry. The Hash and Size attributes are set on // calling Close() on the Writer. func CreateWriter(options *CreateOptions) (io.WriteCloser, *Entry, error) { - o, err := getValidOptions(options) + o, err := getValidCreateOptions(options) if err != nil { return nil, nil, err } @@ -252,7 +252,7 @@ func createHardLink(o *CreateOptions) error { return err } -func getValidOptions(options *CreateOptions) (*CreateOptions, error) { +func getValidCreateOptions(options *CreateOptions) (*CreateOptions, error) { optsCopy := *options o := &optsCopy if o.Root == "" { diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go index 4276d8a6..a69a8b0a 100644 --- a/internal/fsutil/move.go +++ b/internal/fsutil/move.go @@ -21,7 +21,7 @@ type MoveOptions struct { OverrideMode bool } -// Move moves a filesystem entry according to the provided options. +// Move moves or create a filesystem entry according to the provided options. // // Move can return errors from the os package. func Move(options *MoveOptions) error { @@ -50,13 +50,13 @@ func Move(options *MoveOptions) error { case fs.ModeSymlink: err = os.Rename(srcPath, dstPath) case fs.ModeDir: - createOptions := &CreateOptions{ + err = createDir(&CreateOptions{ Root: o.DstRoot, Path: o.Path, Mode: o.Mode, OverrideMode: o.OverrideMode, } - err = createDir(createOptions) +) default: err = fmt.Errorf("unsupported file type: %s", o.Path) } diff --git a/internal/fsutil/remove.go b/internal/fsutil/remove.go index 6a2c731c..c7884404 100644 --- a/internal/fsutil/remove.go +++ b/internal/fsutil/remove.go @@ -14,6 +14,10 @@ type RemoveOptions struct { Path string } +// Remove removes a filesystem entry according to the provided options. +// Non-empty directories are not removed. +// +// Remove can return errors from the os and syscall packages. func Remove(options *RemoveOptions) error { options, err := getValidRemoveOptions(options) if err != nil { From 27a0d229444031cd70c752afb12950d32f5db8ba Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 09:42:38 +0100 Subject: [PATCH 12/65] style: fix lint error --- internal/fsutil/move.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go index a69a8b0a..1a426633 100644 --- a/internal/fsutil/move.go +++ b/internal/fsutil/move.go @@ -55,8 +55,7 @@ func Move(options *MoveOptions) error { Path: o.Path, Mode: o.Mode, OverrideMode: o.OverrideMode, - } -) + }) default: err = fmt.Errorf("unsupported file type: %s", o.Path) } From 5f377b0a06d6cdce476c603332e19428ac1d41e0 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 17:01:38 +0100 Subject: [PATCH 13/65] tests: Move and Remove --- internal/deb/extract_test.go | 2 +- internal/fsutil/create.go | 2 +- internal/fsutil/create_test.go | 4 +- internal/fsutil/move.go | 3 +- internal/fsutil/move_test.go | 258 +++++++++++++++++++++++++++++++++ internal/fsutil/remove.go | 9 +- internal/fsutil/remove_test.go | 145 ++++++++++++++++++ internal/slicer/slicer_test.go | 2 +- 8 files changed, 415 insertions(+), 10 deletions(-) create mode 100644 internal/fsutil/move_test.go create mode 100644 internal/fsutil/remove_test.go diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 1ec0b8a5..30f77816 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -485,7 +485,7 @@ var extractTests = []extractTest{{ }}, }, }, - error: `cannot extract from package "test-package": cannot create path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, + error: `cannot extract from package "test-package": cannot handle path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, }} func (s *S) TestExtract(c *C) { diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index e4987e64..3cd64708 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -268,7 +268,7 @@ func getValidCreateOptions(options *CreateOptions) (*CreateOptions, error) { func absPath(root, relPath string) (string, error) { path := filepath.Clean(filepath.Join(root, relPath)) if !strings.HasPrefix(path, root) { - return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) + return "", fmt.Errorf("cannot handle path %s outside of root %s", path, root) } return path, nil } diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index 5e908fcc..370568c0 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -279,7 +279,7 @@ var createTests = []createTest{{ Mode: 0666, Data: bytes.NewBufferString("hijacking system file"), }, - error: `cannot create path /file outside of root /root/`, + error: `cannot handle path /file outside of root /root/`, }, { summary: "Hardlink cannot escape Root", options: fsutil.CreateOptions{ @@ -410,7 +410,7 @@ var createWriterTests = []createWriterTest{{ Mode: 0644, MakeParents: true, }, - error: `cannot create path /file outside of root /root/`, + error: `cannot handle path /file outside of root /root/`, }} func (s *S) TestCreateWriter(c *C) { diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go index 1a426633..e47f135d 100644 --- a/internal/fsutil/move.go +++ b/internal/fsutil/move.go @@ -46,8 +46,7 @@ func Move(options *MoveOptions) error { } switch o.Mode & fs.ModeType { - case 0: - case fs.ModeSymlink: + case 0, fs.ModeSymlink: err = os.Rename(srcPath, dstPath) case fs.ModeDir: err = createDir(&CreateOptions{ diff --git a/internal/fsutil/move_test.go b/internal/fsutil/move_test.go new file mode 100644 index 00000000..6b68ce09 --- /dev/null +++ b/internal/fsutil/move_test.go @@ -0,0 +1,258 @@ +package fsutil_test + +import ( + "io/fs" + "os" + "path/filepath" + "syscall" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/testutil" +) + +type moveTest struct { + summary string + options fsutil.MoveOptions + hackopt func(c *C, dir string, opts *fsutil.MoveOptions) + result map[string]string + error string +} + +var moveTests = []moveTest{{ + summary: "Move a file and create its parent directory", + options: fsutil.MoveOptions{ + Path: "bar", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/bar": "file 0644 3a6eb079", + }, +}, { + summary: "Move a symlink", + options: fsutil.MoveOptions{ + Path: "foo", + Mode: fs.ModeSymlink, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/baz"), []byte("data"), 0o644), IsNil) + c.Assert(os.Symlink("baz", filepath.Join(dir, "src/foo")), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/src/baz": "file 0644 3a6eb079", + "/dst/": "dir 0755", + "/dst/foo": "symlink baz", + }, +}, { + summary: "Move (create) a directory", + options: fsutil.MoveOptions{ + Path: "foo/", + Mode: fs.ModeDir | 0o765, + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/foo/": "dir 0765", + }, +}, { + summary: "Move (create) a directory with sticky bit", + options: fsutil.MoveOptions{ + Path: "foo", + Mode: fs.ModeDir | fs.ModeSticky | 0o775, + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/foo/": "dir 01775", + }, +}, { + summary: "Cannot move (create) a parent directory without MakeParents set", + options: fsutil.MoveOptions{ + Path: "foo/bar", + Mode: fs.ModeDir | 0o775, + }, + error: `mkdir /[^ ]*/foo/bar: no such file or directory`, +}, { + summary: "Moving to an existing directory keeps the original mode", + options: fsutil.MoveOptions{ + Path: "foo", + Mode: fs.ModeDir | 0o775, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + // mode is not updated. + "/dst/foo/": "dir 0765", + }, +}, { + summary: "Moving to an existing directory overrides the mode is requested", + options: fsutil.MoveOptions{ + Path: "foo", + Mode: fs.ModeDir | 0o775, + OverrideMode: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + // mode is not updated. + "/dst/foo/": "dir 0775", + }, +}, { + summary: "Moving to an existing file overrides the original mode", + options: fsutil.MoveOptions{ + Path: "foo", + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/foo"), []byte("data"), 0o644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "dst/foo"), []byte("data"), 0o666), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/foo": "file 0644 3a6eb079", + }, +}, { + summary: "Move a hard link", + options: fsutil.MoveOptions{ + Path: "hardlink", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) + c.Assert(os.Link(filepath.Join(dir, "src/file"), filepath.Join(dir, "src/hardlink")), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/src/file": "file 0644 3a6eb079 <1>", + "/dst/hardlink": "file 0644 3a6eb079 <1>", + }, +}, { + summary: "No error if hard link already exists", + options: fsutil.MoveOptions{ + Path: "hardlink", + Mode: 0o644, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "dst/foo"), []byte("data"), 0o644), IsNil) + c.Assert(os.Link(filepath.Join(dir, "src/file"), filepath.Join(dir, "src/hardlink")), IsNil) + c.Assert(os.Link(filepath.Join(dir, "dst/foo"), filepath.Join(dir, "dst/hardlink")), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/src/file": "file 0644 3a6eb079 <1>", + "/dst/foo": "file 0644 3a6eb079", + "/dst/hardlink": "file 0644 3a6eb079 <1>", + }, +}, { + summary: "Override a symlink", + options: fsutil.MoveOptions{ + Path: "foo", + Mode: 0o666 | fs.ModeSymlink, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/baz"), []byte("data"), 0o666), IsNil) + c.Assert(os.Symlink("baz", filepath.Join(dir, "src/foo")), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "dst/bar"), []byte("data"), 0o644), IsNil) + c.Assert(os.Symlink("bar", filepath.Join(dir, "dst/foo")), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/src/baz": "file 0666 3a6eb079", + "/dst/": "dir 0755", + "/dst/bar": "file 0644 3a6eb079", + "/dst/foo": "symlink baz", + }, +}, { + summary: "Cannot move file from outside of Root", + options: fsutil.MoveOptions{ + SrcRoot: "/rootsrc", + DstRoot: "/rootdst", + Path: "../file", + Mode: 0o666, + }, + error: `cannot handle path /file outside of root /rootsrc`, +}, { + summary: "Cannot move inexistent file", + options: fsutil.MoveOptions{ + Path: "file", + Mode: 0o666, + }, + error: `rename .*/src/file .*/dst/file: no such file or directory`, +}, { + summary: "Cannot move file to outside of Root", + options: fsutil.MoveOptions{ + SrcRoot: "/rootsrc", + DstRoot: "/rootdst", + Path: "../rootsrc/file", + Mode: 0o666, + }, + error: `cannot handle path /rootsrc/file outside of root /rootdst`, +}} + +func (s *S) TestMove(c *C) { + oldUmask := syscall.Umask(0) + defer func() { + syscall.Umask(oldUmask) + }() + + for _, test := range moveTests { + c.Logf("Test: %s", test.summary) + if test.result == nil { + // Empty map for no files moved. + test.result = make(map[string]string) + } + c.Logf("Options: %v", test.options) + dir := c.MkDir() + options := test.options + if options.SrcRoot == "" { + srcRoot := filepath.Join(dir, "src") + c.Assert(os.Mkdir(srcRoot, 0o755), IsNil) + options.SrcRoot = srcRoot + } + if options.DstRoot == "" { + dstRoot := filepath.Join(dir, "dst") + c.Assert(os.Mkdir(dstRoot, 0o755), IsNil) + options.DstRoot = dstRoot + } + if test.hackopt != nil { + test.hackopt(c, dir, &options) + } + err := fsutil.Move(&options) + + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + + c.Assert(err, IsNil) + c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) + } +} + +func (s *S) TestMoveEmptyRoot(c *C) { + options := &fsutil.MoveOptions{SrcRoot: "", DstRoot: "foo/"} + err := fsutil.Move(options) + c.Assert(err, ErrorMatches, "internal error: MoveOptions.SrcRoot is unset") + options = &fsutil.MoveOptions{SrcRoot: "foo/", DstRoot: ""} + err = fsutil.Move(options) + c.Assert(err, ErrorMatches, "internal error: MoveOptions.DstRoot is unset") +} diff --git a/internal/fsutil/remove.go b/internal/fsutil/remove.go index c7884404..f53a34ae 100644 --- a/internal/fsutil/remove.go +++ b/internal/fsutil/remove.go @@ -27,14 +27,14 @@ func Remove(options *RemoveOptions) error { if err != nil { return err } - if strings.HasSuffix(path, "/") { + if strings.HasSuffix(options.Path, "/") { err = syscall.Rmdir(path) - if err != nil && err != syscall.ENOTEMPTY { + if err != nil && err != syscall.ENOTEMPTY && err != syscall.ENOENT { return err } } else { err = os.Remove(path) - if err != nil { + if err != nil && !os.IsNotExist(err) { return err } } @@ -47,6 +47,9 @@ func getValidRemoveOptions(options *RemoveOptions) (*RemoveOptions, error) { if o.Root == "" { return nil, fmt.Errorf("internal error: RemoveOptions.Root is unset") } + if o.Path == "" { + return nil, fmt.Errorf("internal error: RemoveOptions.Path is unset") + } if o.Root != "/" { o.Root = filepath.Clean(o.Root) + "/" } diff --git a/internal/fsutil/remove_test.go b/internal/fsutil/remove_test.go new file mode 100644 index 00000000..4b744167 --- /dev/null +++ b/internal/fsutil/remove_test.go @@ -0,0 +1,145 @@ +package fsutil_test + +import ( + "os" + "path/filepath" + "syscall" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/testutil" +) + +type removeTest struct { + summary string + options fsutil.RemoveOptions + hackopt func(c *C, dir string, opts *fsutil.RemoveOptions) + result map[string]string + error string +} + +var removeTests = []removeTest{{ + summary: "Remove a file", + options: fsutil.RemoveOptions{ + Path: "file", + }, + hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{}, +}, { + summary: "Remove a non-existent file", + options: fsutil.RemoveOptions{ + Path: "file", + }, + result: map[string]string{}, +}, { + summary: "Remove a non-existent directory", + options: fsutil.RemoveOptions{ + Path: "foo/", + }, + result: map[string]string{}, +}, { + summary: "Remove an empty directory", + options: fsutil.RemoveOptions{ + Path: "foo/bar", + }, + hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { + c.Assert(os.MkdirAll(filepath.Join(dir, "foo/bar"), 0o755), IsNil) + }, + result: map[string]string{ + "/foo/": "dir 0755", + }, +}, { + summary: "Do not remove non-empty directory", + options: fsutil.RemoveOptions{ + Path: "foo/", + }, + hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { + c.Assert(os.MkdirAll(filepath.Join(dir, "foo"), 0o755), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "foo/file"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{ + "/foo/": "dir 0755", + "/foo/file": "file 0644 3a6eb079", + }, +}, { + summary: "Remove a symlink and not the target", + options: fsutil.RemoveOptions{ + Path: "bar", + }, + hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0o644), IsNil) + c.Assert(os.Symlink("foo", filepath.Join(dir, "bar")), IsNil) + }, + result: map[string]string{ + "/foo": "file 0644 3a6eb079", + }, +}, { + summary: "Remove a hard link", + options: fsutil.RemoveOptions{ + Path: "hardlink1", + }, + hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644), IsNil) + c.Assert(os.Link(filepath.Join(dir, "file"), filepath.Join(dir, "hardlink1")), IsNil) + c.Assert(os.Link(filepath.Join(dir, "file"), filepath.Join(dir, "hardlink2")), IsNil) + }, + result: map[string]string{ + "/file": "file 0644 3a6eb079 <1>", + "/hardlink2": "file 0644 3a6eb079 <1>", + }, +}, { + summary: "Cannot remove file outside of Root", + options: fsutil.RemoveOptions{ + Root: "/root", + Path: "../file", + }, + error: `cannot handle path /file outside of root /root/`, +}} + +func (s *S) TestRemove(c *C) { + oldUmask := syscall.Umask(0) + defer func() { + syscall.Umask(oldUmask) + }() + + for _, test := range removeTests { + c.Logf("Test: %s", test.summary) + if test.result == nil { + // Empty map for no files left. + test.result = make(map[string]string) + } + c.Logf("Options: %v", test.options) + dir := c.MkDir() + options := test.options + if options.Root == "" { + options.Root = dir + } + if test.hackopt != nil { + test.hackopt(c, dir, &options) + } + err := fsutil.Remove(&options) + + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + + c.Assert(err, IsNil) + c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) + } +} + +func (s *S) TestRemoveEmptyRoot(c *C) { + options := &fsutil.RemoveOptions{Root: ""} + err := fsutil.Remove(options) + c.Assert(err, ErrorMatches, "internal error: RemoveOptions.Root is unset") +} + +func (s *S) TestRemoveEmptyPath(c *C) { + options := &fsutil.RemoveOptions{Root: "foo/"} + err := fsutil.Remove(options) + c.Assert(err, ErrorMatches, "internal error: RemoveOptions.Path is unset") +} diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index fe408e6d..d031393a 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1806,7 +1806,7 @@ var slicerTests = []slicerTest{{ /**: `, }, - error: `cannot extract from package "test-package": cannot create path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, + error: `cannot extract from package "test-package": cannot handle path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ From f74265ca42e02eb453e0e8181661f599f67309db Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 17:22:45 +0100 Subject: [PATCH 14/65] tests: FindPathsInRelease --- internal/manifestutil/manifestutil_test.go | 112 +++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 2bab0a68..c0f35886 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -110,10 +110,122 @@ func (s *S) TestFindPaths(c *C) { } } +var findPathsInReleaseTests = []struct { + summary string + release *setup.Release + expected []string +}{{ + summary: "Single package with single slice", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + }, + }, + expected: []string{"/folder/manifest.wall"}, +}, { + summary: "No slices with generate:manifest", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{}, + }, + }, + }, + }, + }, + expected: []string{}, +}, { + summary: "Multiple packages with multiple slices", + release: &setup.Release{ + Packages: map[string]*setup.Package{ + "package1": { + Name: "package1", + Slices: map[string]*setup.Slice{ + "slice1": { + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + "slice2": { + Name: "slice2", + Contents: map[string]setup.PathInfo{ + "/folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + "package2": { + Name: "package2", + Slices: map[string]*setup.Slice{ + "slice3": { + Name: "slice3", + Contents: map[string]setup.PathInfo{}, + }, + "slice4": { + Name: "slice4", + Contents: map[string]setup.PathInfo{ + "/other-folder/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + }, + }, + }, + }, + }, + // Note: /folder/manifest.wall appears twice because both slice1 and slice2 declare it + expected: []string{"/folder/manifest.wall", "/folder/manifest.wall", "/other-folder/manifest.wall"}, +}, { + summary: "Empty release", + release: &setup.Release{ + Packages: map[string]*setup.Package{}, + }, + expected: []string{}, +}} + +func (s *S) TestFindPathsInRelease(c *C) { + for _, test := range findPathsInReleaseTests { + c.Logf("Summary: %s", test.summary) + + manifestPaths := manifestutil.FindPathsInRelease(test.release) + + c.Assert(manifestPaths, HasLen, len(test.expected)) + slices.Sort(manifestPaths) + slices.Sort(test.expected) + c.Assert(manifestPaths, DeepEquals, test.expected) + } +} + var slice1 = &setup.Slice{ Package: "package1", Name: "slice1", } + var slice2 = &setup.Slice{ Package: "package2", Name: "slice2", From 799373c14c8599214389b853d9b20de664555238 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 17:38:49 +0100 Subject: [PATCH 15/65] tests: more test cases for Move --- internal/fsutil/move_test.go | 82 ++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/internal/fsutil/move_test.go b/internal/fsutil/move_test.go index 6b68ce09..4b5c23b5 100644 --- a/internal/fsutil/move_test.go +++ b/internal/fsutil/move_test.go @@ -35,6 +35,35 @@ var moveTests = []moveTest{{ "/dst/": "dir 0755", "/dst/bar": "file 0644 3a6eb079", }, +}, { + summary: "Move a file when parent directory exists", + options: fsutil.MoveOptions{ + Path: "bar", + Mode: 0o644, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/bar": "file 0644 3a6eb079", + }, +}, { + summary: "Move an empty file", + options: fsutil.MoveOptions{ + Path: "empty", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/empty"), []byte(""), 0o644), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/empty": "file 0644 empty", + }, }, { summary: "Move a symlink", options: fsutil.MoveOptions{ @@ -190,6 +219,59 @@ var moveTests = []moveTest{{ Mode: 0o666, }, error: `cannot handle path /file outside of root /rootsrc`, +}, { + summary: "Path with ./ component", + options: fsutil.MoveOptions{ + Path: "./file", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/file": "file 0644 3a6eb079", + }, +}, { + summary: "Path with ../ component normalizes correctly", + options: fsutil.MoveOptions{ + Path: "foo/../bar", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) + }, + result: map[string]string{ + "/src/": "dir 0755", + "/dst/": "dir 0755", + "/dst/bar": "file 0644 3a6eb079", + }, +}, { + summary: "Cannot move to a path where parent is a file", + options: fsutil.MoveOptions{ + Path: "file/subpath", + Mode: 0o644, + MakeParents: true, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "dst/file"), []byte("data"), 0o644), IsNil) + }, + error: `mkdir .*/dst/file: not a directory`, +}, { + summary: "Cannot move a file to overwrite a directory", + options: fsutil.MoveOptions{ + Path: "target", + Mode: 0o644, + }, + hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { + c.Assert(os.WriteFile(filepath.Join(dir, "src/target"), []byte("data"), 0o644), IsNil) + c.Assert(os.Mkdir(filepath.Join(dir, "dst/target"), 0o755), IsNil) + }, + error: `rename .*/src/target .*/dst/target: file exists`, }, { summary: "Cannot move inexistent file", options: fsutil.MoveOptions{ From dbe3f309f3198ff4163b36639dcc81be37fd7904 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 3 Feb 2026 17:52:41 +0100 Subject: [PATCH 16/65] tests: add spread test --- tests/recut/task.yaml | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/recut/task.yaml diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml new file mode 100644 index 00000000..0a1a09d6 --- /dev/null +++ b/tests/recut/task.yaml @@ -0,0 +1,57 @@ +summary: Recut reuses manifest to restore previous slices + +execute: | + rootfs_folder=rootfs_${RELEASE} + mkdir -p $rootfs_folder + + chisel_release="./release_${RELEASE}" + mkdir -p ${chisel_release}/slices + + ref_chisel_release="ref-chisel-release_${RELEASE}" + git clone --depth=1 -b ${OS}-${RELEASE} \ + https://github.com/canonical/chisel-releases $ref_chisel_release + + cp ${ref_chisel_release}/chisel.yaml ${chisel_release}/chisel.yaml + + cat >${chisel_release}/slices/base-files.yaml <<'EOF' + package: base-files + + slices: + slice-a: + contents: + /etc/hostname: + slice-b: + contents: + /etc/issue: + manifest: + contents: + /chisel/**: {generate: manifest} + EOF + + EXTRA_OPTIONS="" + if [ "$RELEASE" = "23.10" -o "$RELEASE" = "20.04" ]; then + EXTRA_OPTIONS="--ignore=unmaintained " + fi + + # First cut generates manifest and installs slice-a + chisel cut --release $chisel_release \ + --root $rootfs_folder \ + $EXTRA_OPTIONS \ + base-files_slice-a base-files_manifest + + test -s ${rootfs_folder}/etc/hostname + test -f ${rootfs_folder}/chisel/manifest.wall + + # Remove a file from the first slice + rm -f ${rootfs_folder}/etc/hostname + test ! -e ${rootfs_folder}/etc/hostname + + # Second cut only requests slice-b, slice-a should be restored via manifest + chisel cut --release $chisel_release \ + --root $rootfs_folder \ + $EXTRA_OPTIONS \ + base-files_slice-b + + test -s ${rootfs_folder}/etc/hostname + test -s ${rootfs_folder}/etc/issue + test -f ${rootfs_folder}/chisel/manifest.wall From becfe42d306a3a9f37f1b7c22eb7d769ad0765ac Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 08:35:55 +0100 Subject: [PATCH 17/65] tests: fix recut spread test --- tests/recut/task.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index 0a1a09d6..a01edcb9 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -1,4 +1,4 @@ -summary: Recut reuses manifest to restore previous slices +summary: Recut relies on manifest to update existing content execute: | rootfs_folder=rootfs_${RELEASE} @@ -19,7 +19,7 @@ execute: | slices: slice-a: contents: - /etc/hostname: + /etc/debian_version: slice-b: contents: /etc/issue: @@ -39,12 +39,11 @@ execute: | $EXTRA_OPTIONS \ base-files_slice-a base-files_manifest - test -s ${rootfs_folder}/etc/hostname + test -s ${rootfs_folder}/etc/debian_version test -f ${rootfs_folder}/chisel/manifest.wall # Remove a file from the first slice - rm -f ${rootfs_folder}/etc/hostname - test ! -e ${rootfs_folder}/etc/hostname + rm -f ${rootfs_folder}/etc/debian_version # Second cut only requests slice-b, slice-a should be restored via manifest chisel cut --release $chisel_release \ @@ -52,6 +51,6 @@ execute: | $EXTRA_OPTIONS \ base-files_slice-b - test -s ${rootfs_folder}/etc/hostname + test -s ${rootfs_folder}/etc/debian_version test -s ${rootfs_folder}/etc/issue test -f ${rootfs_folder}/chisel/manifest.wall From 3c994b856ad7af5b9ba35f3b1dd8500bee82a631 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 09:44:11 +0100 Subject: [PATCH 18/65] tests: SelectValidManifest --- internal/slicer/slicer_test.go | 206 +++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d031393a..3d921e9e 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2017,6 +2017,118 @@ func (s *S) TestRun(c *C) { runSlicerTests(s, c, v2FormatTests) } +type selectValidManifestTest struct { + summary string + build func() *setup.Release + setup func(c *C, targetDir string, release *setup.Release) + noMatch bool + error string +} + +var selectValidManifestTests = []selectValidManifestTest{{ + summary: "No manifest paths in release", + build: func() *setup.Release { + return &setup.Release{Packages: map[string]*setup.Package{}} + }, + noMatch: true, +}, { + summary: "Manifest path missing in target", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + noMatch: true, +}, { + summary: "Valid manifest selected", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeManifest(c, targetDir, manifestPath, releaseManifestSlice(release), "hash1") + }, +}, { + summary: "Two consistent manifests are accepted", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := releaseManifestSlice(release) + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeManifest(c, targetDir, manifestPathB, slice, "hash1") + }, +}, { + summary: "Inconsistent manifests with same schema are rejected", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := releaseManifestSlice(release) + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeManifest(c, targetDir, manifestPathB, slice, "hash2") + }, + error: `inconsistent manifests: ".*" and ".*"`, +}, { + summary: "Invalid manifest data returns error", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := filepath.Join(targetDir, manifestPathForDir("/chisel/**")) + err := os.MkdirAll(filepath.Dir(manifestPath), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(manifestPath, []byte("not-a-zstd-manifest"), 0o644) + c.Assert(err, IsNil) + }, + error: "cannot read manifest: invalid input: .*", +}, { + summary: "Manifest validation error is returned", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeInvalidManifest(c, targetDir, manifestPath) + }, + error: `invalid manifest: path /file has no matching entry in contents`, +}, { + summary: "Manifest read fails on invalid schema", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeInvalidSchemaManifest(c, targetDir, manifestPath) + }, + error: `cannot read manifest: unknown schema version "9.9"`, +}} + +func (s *S) TestSelectValidManifest(c *C) { + for _, test := range selectValidManifestTests { + c.Logf("Summary: %s", test.summary) + release := test.build() + targetDir := c.MkDir() + if test.setup != nil { + test.setup(c, targetDir, release) + } + mfest, err := slicer.SelectValidManifest(targetDir, release) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + if test.noMatch { + c.Assert(mfest, IsNil) + continue + } + c.Assert(mfest, NotNil) + c.Assert(mfest.Schema(), Equals, manifest.Schema) + } +} + func runSlicerTests(s *S, c *C, tests []slicerTest) { for _, test := range tests { for _, testSlices := range testutil.Permutations(test.slices) { @@ -2165,6 +2277,36 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { } } +func buildReleaseWithManifestDirs(dirs ...string) *setup.Release { + contents := map[string]setup.PathInfo{} + for _, dir := range dirs { + contents[dir] = setup.PathInfo{Kind: "generate", Generate: "manifest"} + } + return &setup.Release{ + Packages: map[string]*setup.Package{ + "test-package": { + Name: "test-package", + Slices: map[string]*setup.Slice{ + "manifest": { + Package: "test-package", + Name: "manifest", + Contents: contents, + }, + }, + }, + }, + } +} + +func releaseManifestSlice(release *setup.Release) *setup.Slice { + return release.Packages["test-package"].Slices["manifest"] +} + +func manifestPathForDir(dir string) string { + base := strings.TrimSuffix(dir, "**") + return path.Join(base, manifestutil.DefaultFilename) +} + func treeDumpManifestPaths(mfest *manifest.Manifest) (map[string]string, error) { result := make(map[string]string) err := mfest.IteratePaths("", func(path *manifest.Path) error { @@ -2241,3 +2383,67 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { return mfest } + +func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + options := &manifestutil.WriteOptions{ + PackageInfo: []*archive.PackageInfo{{ + Name: slice.Package, + Version: "1.0", + Arch: "amd64", + SHA256: "pkg-hash", + }}, + Selection: []*setup.Slice{slice}, + Report: &manifestutil.Report{Root: "/", Entries: map[string]manifestutil.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0o644, + SHA256: hash, + Size: 3, + Slices: map[*setup.Slice]bool{slice: true}, + }, + }}, + } + err = manifestutil.Write(options, zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) +} + +func writeInvalidManifest(c *C, targetDir, manifestPath string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: manifest.Schema}) + err = dbw.Add(&manifest.Path{Kind: "path", Path: "/file", Mode: "0644"}) + c.Assert(err, IsNil) + _, err = dbw.WriteTo(zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) +} + +func writeInvalidSchemaManifest(c *C, targetDir, manifestPath string) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.Create(mfestPath) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: "9.9"}) + _, err = dbw.WriteTo(zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) +} From 6ebb893978f269a4f8d906a7f242dc5deb2295df Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 09:47:25 +0100 Subject: [PATCH 19/65] fix: add missing deps --- internal/slicer/slicer_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 3d921e9e..c8f7c885 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -19,6 +19,7 @@ import ( "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" + "github.com/canonical/chisel/public/jsonwall" "github.com/canonical/chisel/public/manifest" ) From ceac217b9da0559af0d3656bc45ca8982af49ceb Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 11:37:59 +0100 Subject: [PATCH 20/65] tests: recut feature --- internal/slicer/slicer_test.go | 180 +++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index c8f7c885..17240857 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -15,6 +15,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" @@ -34,6 +35,7 @@ type slicerTest struct { pkgs []*testutil.TestPackage slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) + prefill func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) filesystem map[string]string manifestPaths map[string]string manifestPkgs map[string]string @@ -1979,6 +1981,147 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, +}, { + summary: "Recut removes obsolete paths when selection shrinks", + slices: []setup.SliceKey{{"test-package", "slice2"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /foo: {text: data1} + slice2: + contents: + /bar: {text: data2} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + pkg := release.Packages["test-package"] + slice1 := pkg.Slices["slice1"] + slice2 := pkg.Slices["slice2"] + manifestSlice := pkg.Slices["manifest"] + writeFile(c, opts.TargetDir, "/foo", []byte("data1"), 0o644) + writeFile(c, opts.TargetDir, "/bar", []byte("data2"), 0o644) + report, err := manifestutil.NewReport(opts.TargetDir) + c.Assert(err, IsNil) + err = report.Add(slice1, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/foo"), + Mode: 0o644, + SHA256: "5b41362b", + Size: 5, + }) + c.Assert(err, IsNil) + err = report.Add(slice2, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/bar"), + Mode: 0o644, + SHA256: "d98cf53e", + Size: 5, + }) + c.Assert(err, IsNil) + err = report.Add(manifestSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, manifestPath), + Mode: 0o644, + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, slice2, manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/bar": "file 0644 d98cf53e", + }, + manifestPaths: map[string]string{ + "/bar": "file 0644 d98cf53e {test-package_slice2}", + }, +}, { + summary: "Recut restores modified content and mode", + slices: []setup.SliceKey{{"test-package", "slice1"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /file: {text: data1} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + pkg := release.Packages["test-package"] + slice1 := pkg.Slices["slice1"] + manifestSlice := pkg.Slices["manifest"] + writeFile(c, opts.TargetDir, "/file", []byte("data1"), 0o644) + report, err := manifestutil.NewReport(opts.TargetDir) + c.Assert(err, IsNil) + err = report.Add(slice1, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/file"), + Mode: 0o644, + SHA256: "5b41362b", + Size: 5, + }) + c.Assert(err, IsNil) + err = report.Add(manifestSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, manifestPath), + Mode: 0o644, + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, manifestSlice}, report) + modifiedPath := filepath.Join(opts.TargetDir, "file") + err = os.WriteFile(modifiedPath, []byte("data2"), 0o700) + c.Assert(err, IsNil) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/file": "file 0644 5b41362b", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 5b41362b {test-package_slice1}", + }, +}, { + summary: "Recut keeps untracked files", + slices: []setup.SliceKey{{"test-package", "slice1"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + slice1: + contents: + /file: {text: data1} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + pkg := release.Packages["test-package"] + slice1 := pkg.Slices["slice1"] + manifestSlice := pkg.Slices["manifest"] + writeFile(c, opts.TargetDir, "/file", []byte("data1"), 0o644) + report, err := manifestutil.NewReport(opts.TargetDir) + c.Assert(err, IsNil) + err = report.Add(slice1, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/file"), + Mode: 0o644, + SHA256: "5b41362b", + Size: 5, + }) + c.Assert(err, IsNil) + err = report.Add(manifestSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, manifestPath), + Mode: 0o644, + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, manifestSlice}, report) + err = os.MkdirAll(filepath.Join(opts.TargetDir, "extra"), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(opts.TargetDir, "extra", "untracked"), []byte("data"), 0o644) + c.Assert(err, IsNil) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/extra/": "dir 0755", + "/extra/untracked": "file 0644 3a6eb079", + "/file": "file 0644 5b41362b", + }, + manifestPaths: map[string]string{ + "/file": "file 0644 5b41362b {test-package_slice1}", + }, }} func (s *S) TestRun(c *C) { @@ -2224,6 +2367,9 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { if test.hackopt != nil { test.hackopt(c, &options) } + if test.prefill != nil { + test.prefill(c, &options, release, manifestPath) + } err = slicer.Run(&options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) @@ -2415,6 +2561,32 @@ func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, has c.Assert(err, IsNil) c.Assert(zw.Close(), IsNil) c.Assert(f.Close(), IsNil) + c.Assert(os.Chmod(mfestPath, 0o644), IsNil) +} + +func writeManifestReport(c *C, targetDir, manifestPath, pkgName string, selection []*setup.Slice, report *manifestutil.Report) { + mfestPath := filepath.Join(targetDir, manifestPath) + err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) + c.Assert(err, IsNil) + f, err := os.OpenFile(mfestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) + c.Assert(err, IsNil) + zw, err := zstd.NewWriter(f) + c.Assert(err, IsNil) + options := &manifestutil.WriteOptions{ + PackageInfo: []*archive.PackageInfo{{ + Name: pkgName, + Version: "1.0", + Arch: "amd64", + SHA256: "pkg-hash", + }}, + Selection: selection, + Report: report, + } + err = manifestutil.Write(options, zw) + c.Assert(err, IsNil) + c.Assert(zw.Close(), IsNil) + c.Assert(f.Close(), IsNil) + c.Assert(os.Chmod(mfestPath, 0o644), IsNil) } func writeInvalidManifest(c *C, targetDir, manifestPath string) { @@ -2448,3 +2620,11 @@ func writeInvalidSchemaManifest(c *C, targetDir, manifestPath string) { c.Assert(zw.Close(), IsNil) c.Assert(f.Close(), IsNil) } + +func writeFile(c *C, targetDir, relPath string, data []byte, mode fs.FileMode) { + path := filepath.Join(targetDir, relPath) + err := os.MkdirAll(filepath.Dir(path), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(path, data, mode) + c.Assert(err, IsNil) +} From 7e34e3201199517dab0b74fbc189bb97032e249e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 13:35:46 +0100 Subject: [PATCH 21/65] fix: avoid duplicates in FindPathsInRelease --- internal/manifestutil/manifestutil.go | 7 ++++--- internal/manifestutil/manifestutil_test.go | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index b2946d57..4ace8880 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/fs" + "maps" "path/filepath" "slices" "sort" @@ -44,16 +45,16 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { // FindPathsInRelease finds all the paths marked with "generate:manifest" // for the given release. func FindPathsInRelease(r *setup.Release) []string { - manifestPaths := make([]string, 0) + manifestPaths := make(map[string]struct{}) collector := func(path string, slice *setup.Slice) { - manifestPaths = append(manifestPaths, path) + manifestPaths[path] = struct{}{} } for _, pkg := range r.Packages { for _, slice := range pkg.Slices { collectManifests(slice, collector) } } - return manifestPaths + return slices.Sorted(maps.Keys(manifestPaths)) } type WriteOptions struct { diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index c0f35886..016c1c5a 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -150,7 +150,6 @@ var findPathsInReleaseTests = []struct { }, }, }, - expected: []string{}, }, { summary: "Multiple packages with multiple slices", release: &setup.Release{ @@ -199,13 +198,12 @@ var findPathsInReleaseTests = []struct { }, }, // Note: /folder/manifest.wall appears twice because both slice1 and slice2 declare it - expected: []string{"/folder/manifest.wall", "/folder/manifest.wall", "/other-folder/manifest.wall"}, + expected: []string{"/folder/manifest.wall", "/other-folder/manifest.wall"}, }, { summary: "Empty release", release: &setup.Release{ Packages: map[string]*setup.Package{}, }, - expected: []string{}, }} func (s *S) TestFindPathsInRelease(c *C) { From e1d4e17b54a9bbf0d6cbcf23cc73b98f937abca7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 4 Feb 2026 15:34:17 +0100 Subject: [PATCH 22/65] ci: rerun From c3db9d428f86d5b97e47e6c2a3b725026fc012ed Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 5 Feb 2026 08:46:27 +0100 Subject: [PATCH 23/65] fix: apply PR suggestions --- internal/fsutil/create.go | 23 ++++++++--------------- internal/fsutil/move.go | 11 ++++------- internal/fsutil/move_test.go | 20 ++------------------ internal/fsutil/remove.go | 15 ++++----------- internal/slicer/slicer.go | 3 +-- 5 files changed, 19 insertions(+), 53 deletions(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 3cd64708..59016145 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -166,22 +166,15 @@ func createDir(o *CreateOptions) error { if err != nil { return err } - fileinfo, err := os.Lstat(path) - if err == nil { - if fileinfo.IsDir() { - if fileinfo.Mode() != o.Mode && o.OverrideMode { - return os.Chmod(path, o.Mode) - } - return nil - } - err = os.Remove(path) - if err != nil { - return err - } - } else if !os.IsNotExist(err) { - return err + err = os.Mkdir(path, o.Mode) + if os.IsExist(err) { + // TODO: Detect if existing content is a file. ErrExist is also returned + // if a file exists at this path, so returning nil here creates an + // inconsistency between our view of the content and the real content on + // disk. + return nil } - return os.Mkdir(path, o.Mode) + return err } func createFile(o *CreateOptions) error { diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go index e47f135d..b0ca4623 100644 --- a/internal/fsutil/move.go +++ b/internal/fsutil/move.go @@ -10,18 +10,15 @@ import ( type MoveOptions struct { SrcRoot string DstRoot string - // Path is relative to Root. + // Path is relative to SrcRoot. Path string Mode fs.FileMode // If MakeParents is true, missing parent directories of Path are - // created with permissions 0755. + // created with permissions 0755 in DstRoot. MakeParents bool - // If OverrideMode is true and entry already exists, update the mode. Does - // not affect symlinks. - OverrideMode bool } -// Move moves or create a filesystem entry according to the provided options. +// Move moves or creates a filesystem entry according to the provided options. // // Move can return errors from the os package. func Move(options *MoveOptions) error { @@ -53,7 +50,7 @@ func Move(options *MoveOptions) error { Root: o.DstRoot, Path: o.Path, Mode: o.Mode, - OverrideMode: o.OverrideMode, + OverrideMode: true, }) default: err = fmt.Errorf("unsupported file type: %s", o.Path) diff --git a/internal/fsutil/move_test.go b/internal/fsutil/move_test.go index 4b5c23b5..906774d5 100644 --- a/internal/fsutil/move_test.go +++ b/internal/fsutil/move_test.go @@ -111,26 +111,10 @@ var moveTests = []moveTest{{ }, error: `mkdir /[^ ]*/foo/bar: no such file or directory`, }, { - summary: "Moving to an existing directory keeps the original mode", - options: fsutil.MoveOptions{ - Path: "foo", - Mode: fs.ModeDir | 0o775, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - // mode is not updated. - "/dst/foo/": "dir 0765", - }, -}, { - summary: "Moving to an existing directory overrides the mode is requested", + summary: "Do not override mode of existing", options: fsutil.MoveOptions{ Path: "foo", Mode: fs.ModeDir | 0o775, - OverrideMode: true, }, hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) @@ -139,7 +123,7 @@ var moveTests = []moveTest{{ "/src/": "dir 0755", "/dst/": "dir 0755", // mode is not updated. - "/dst/foo/": "dir 0775", + "/dst/foo/": "dir 0765", }, }, { summary: "Moving to an existing file overrides the original mode", diff --git a/internal/fsutil/remove.go b/internal/fsutil/remove.go index f53a34ae..4f7cb865 100644 --- a/internal/fsutil/remove.go +++ b/internal/fsutil/remove.go @@ -1,10 +1,10 @@ package fsutil import ( + "errors" "fmt" "os" "path/filepath" - "strings" "syscall" ) @@ -27,16 +27,9 @@ func Remove(options *RemoveOptions) error { if err != nil { return err } - if strings.HasSuffix(options.Path, "/") { - err = syscall.Rmdir(path) - if err != nil && err != syscall.ENOTEMPTY && err != syscall.ENOENT { - return err - } - } else { - err = os.Remove(path) - if err != nil && !os.IsNotExist(err) { - return err - } + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENOTEMPTY) { + return err } return nil } diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2ccab61b..88fce6ad 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -389,14 +389,13 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes Path: path, Mode: entry.Mode, MakeParents: true, - OverrideMode: true, }) if err != nil { return err } } - // Remove missing paths + // Remove missing paths. missingPaths := make([]string, 0) err := mfest.IteratePaths("", func(path *manifest.Path) error { _, ok := report.Entries[path.Path] From e1e9c43c149643156de08e3a1b53da892be8bcfa Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 5 Feb 2026 09:17:57 +0100 Subject: [PATCH 24/65] refactor: simplify FindPaths and FindPathsInRelease --- internal/manifestutil/manifestutil.go | 32 ++++++++++++--------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 4ace8880..63a7be83 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -19,25 +19,18 @@ import ( const DefaultFilename = "manifest.wall" -func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) { - for path, info := range slice.Contents { - if info.Generate == setup.GenerateManifest { - dir := strings.TrimSuffix(path, "**") - path = filepath.Join(dir, DefaultFilename) - collector(path, slice) - } - } -} - // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { manifestSlices := make(map[string][]*setup.Slice) - collector := func(path string, slice *setup.Slice) { - manifestSlices[path] = append(manifestSlices[path], slice) - } for _, slice := range slices { - collectManifests(slice, collector) + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + manifestSlices[path] = append(manifestSlices[path], slice) + } + } } return manifestSlices } @@ -46,12 +39,15 @@ func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { // for the given release. func FindPathsInRelease(r *setup.Release) []string { manifestPaths := make(map[string]struct{}) - collector := func(path string, slice *setup.Slice) { - manifestPaths[path] = struct{}{} - } for _, pkg := range r.Packages { for _, slice := range pkg.Slices { - collectManifests(slice, collector) + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + manifestPaths[path] = struct{}{} + } + } } } return slices.Sorted(maps.Keys(manifestPaths)) From da567ba77449e1a8cf9f80761af387957bd3a425 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 5 Feb 2026 09:19:36 +0100 Subject: [PATCH 25/65] style: lint --- internal/slicer/slicer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 88fce6ad..8322dee3 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -384,11 +384,11 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes for _, path := range paths { entry := report.Entries[path] err := fsutil.Move(&fsutil.MoveOptions{ - SrcRoot: tempDir, - DstRoot: targetDir, - Path: path, - Mode: entry.Mode, - MakeParents: true, + SrcRoot: tempDir, + DstRoot: targetDir, + Path: path, + Mode: entry.Mode, + MakeParents: true, }) if err != nil { return err From 2a6c8913a869baf58c9ff0943e60c10d3c823ae0 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 5 Feb 2026 09:21:59 +0100 Subject: [PATCH 26/65] fix: remove outdated comment --- internal/manifestutil/manifestutil_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 016c1c5a..216c68e2 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -197,7 +197,6 @@ var findPathsInReleaseTests = []struct { }, }, }, - // Note: /folder/manifest.wall appears twice because both slice1 and slice2 declare it expected: []string{"/folder/manifest.wall", "/other-folder/manifest.wall"}, }, { summary: "Empty release", From 398e032d6df1822e7fb37be10fe55a8dadcca813 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 5 Feb 2026 11:28:22 +0100 Subject: [PATCH 27/65] test: fix inaccurate tests --- internal/fsutil/move_test.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/internal/fsutil/move_test.go b/internal/fsutil/move_test.go index 906774d5..e7df92ba 100644 --- a/internal/fsutil/move_test.go +++ b/internal/fsutil/move_test.go @@ -23,31 +23,38 @@ type moveTest struct { var moveTests = []moveTest{{ summary: "Move a file and create its parent directory", options: fsutil.MoveOptions{ - Path: "bar", + Path: "foo/bar", Mode: 0o644, MakeParents: true, }, hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) + c.Assert(os.Mkdir(filepath.Join(dir, "src/foo/"), fs.ModeDir|0o755), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "src/foo/bar"), []byte("data"), 0o644), IsNil) }, result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/bar": "file 0644 3a6eb079", + "/src/": "dir 0755", + "/src/foo/": "dir 0755", + "/dst/": "dir 0755", + "/dst/foo/": "dir 0755", + "/dst/foo/bar": "file 0644 3a6eb079", }, }, { summary: "Move a file when parent directory exists", options: fsutil.MoveOptions{ - Path: "bar", + Path: "foo/bar", Mode: 0o644, }, hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) + c.Assert(os.Mkdir(filepath.Join(dir, "src/foo/"), fs.ModeDir|0o755), IsNil) + c.Assert(os.WriteFile(filepath.Join(dir, "src/foo/bar"), []byte("data"), 0o644), IsNil) + c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o755), IsNil) }, result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/bar": "file 0644 3a6eb079", + "/src/": "dir 0755", + "/src/foo/": "dir 0755", + "/dst/": "dir 0755", + "/dst/foo/": "dir 0755", + "/dst/foo/bar": "file 0644 3a6eb079", }, }, { summary: "Move an empty file", @@ -111,10 +118,10 @@ var moveTests = []moveTest{{ }, error: `mkdir /[^ ]*/foo/bar: no such file or directory`, }, { - summary: "Do not override mode of existing", + summary: "Do not override mode of existing directory", options: fsutil.MoveOptions{ - Path: "foo", - Mode: fs.ModeDir | 0o775, + Path: "foo", + Mode: fs.ModeDir | 0o775, }, hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) From 7c17f1844db8876505ca73bd60153dc23cbf4096 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 6 Feb 2026 14:20:13 +0100 Subject: [PATCH 28/65] refactor: apply PR suggestions --- internal/fsutil/move.go | 81 ----- internal/fsutil/move_test.go | 331 --------------------- internal/fsutil/remove.go | 50 ---- internal/fsutil/remove_test.go | 145 --------- internal/manifestutil/manifestutil_test.go | 2 + internal/slicer/slicer.go | 50 ++-- internal/slicer/slicer_test.go | 295 +++++++++++++++++- tests/recut/task.yaml | 2 +- 8 files changed, 321 insertions(+), 635 deletions(-) delete mode 100644 internal/fsutil/move.go delete mode 100644 internal/fsutil/move_test.go delete mode 100644 internal/fsutil/remove.go delete mode 100644 internal/fsutil/remove_test.go diff --git a/internal/fsutil/move.go b/internal/fsutil/move.go deleted file mode 100644 index b0ca4623..00000000 --- a/internal/fsutil/move.go +++ /dev/null @@ -1,81 +0,0 @@ -package fsutil - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" -) - -type MoveOptions struct { - SrcRoot string - DstRoot string - // Path is relative to SrcRoot. - Path string - Mode fs.FileMode - // If MakeParents is true, missing parent directories of Path are - // created with permissions 0755 in DstRoot. - MakeParents bool -} - -// Move moves or creates a filesystem entry according to the provided options. -// -// Move can return errors from the os package. -func Move(options *MoveOptions) error { - o, err := getValidMoveOptions(options) - if err != nil { - return err - } - - srcPath, err := absPath(options.SrcRoot, o.Path) - if err != nil { - return err - } - dstPath, err := absPath(options.DstRoot, o.Path) - if err != nil { - return err - } - - if o.MakeParents { - if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { - return err - } - } - - switch o.Mode & fs.ModeType { - case 0, fs.ModeSymlink: - err = os.Rename(srcPath, dstPath) - case fs.ModeDir: - err = createDir(&CreateOptions{ - Root: o.DstRoot, - Path: o.Path, - Mode: o.Mode, - OverrideMode: true, - }) - default: - err = fmt.Errorf("unsupported file type: %s", o.Path) - } - if err != nil { - return err - } - - return nil -} - -func getValidMoveOptions(options *MoveOptions) (*MoveOptions, error) { - optsCopy := *options - o := &optsCopy - if o.SrcRoot == "" { - return nil, fmt.Errorf("internal error: MoveOptions.SrcRoot is unset") - } - if o.DstRoot == "" { - return nil, fmt.Errorf("internal error: MoveOptions.DstRoot is unset") - } - if o.SrcRoot != "/" { - o.SrcRoot = filepath.Clean(o.SrcRoot) + "/" - } - if o.DstRoot != "/" { - o.DstRoot = filepath.Clean(o.DstRoot) + "/" - } - return o, nil -} diff --git a/internal/fsutil/move_test.go b/internal/fsutil/move_test.go deleted file mode 100644 index e7df92ba..00000000 --- a/internal/fsutil/move_test.go +++ /dev/null @@ -1,331 +0,0 @@ -package fsutil_test - -import ( - "io/fs" - "os" - "path/filepath" - "syscall" - - . "gopkg.in/check.v1" - - "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/testutil" -) - -type moveTest struct { - summary string - options fsutil.MoveOptions - hackopt func(c *C, dir string, opts *fsutil.MoveOptions) - result map[string]string - error string -} - -var moveTests = []moveTest{{ - summary: "Move a file and create its parent directory", - options: fsutil.MoveOptions{ - Path: "foo/bar", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.Mkdir(filepath.Join(dir, "src/foo/"), fs.ModeDir|0o755), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "src/foo/bar"), []byte("data"), 0o644), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/src/foo/": "dir 0755", - "/dst/": "dir 0755", - "/dst/foo/": "dir 0755", - "/dst/foo/bar": "file 0644 3a6eb079", - }, -}, { - summary: "Move a file when parent directory exists", - options: fsutil.MoveOptions{ - Path: "foo/bar", - Mode: 0o644, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.Mkdir(filepath.Join(dir, "src/foo/"), fs.ModeDir|0o755), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "src/foo/bar"), []byte("data"), 0o644), IsNil) - c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o755), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/src/foo/": "dir 0755", - "/dst/": "dir 0755", - "/dst/foo/": "dir 0755", - "/dst/foo/bar": "file 0644 3a6eb079", - }, -}, { - summary: "Move an empty file", - options: fsutil.MoveOptions{ - Path: "empty", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/empty"), []byte(""), 0o644), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/empty": "file 0644 empty", - }, -}, { - summary: "Move a symlink", - options: fsutil.MoveOptions{ - Path: "foo", - Mode: fs.ModeSymlink, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/baz"), []byte("data"), 0o644), IsNil) - c.Assert(os.Symlink("baz", filepath.Join(dir, "src/foo")), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/src/baz": "file 0644 3a6eb079", - "/dst/": "dir 0755", - "/dst/foo": "symlink baz", - }, -}, { - summary: "Move (create) a directory", - options: fsutil.MoveOptions{ - Path: "foo/", - Mode: fs.ModeDir | 0o765, - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/foo/": "dir 0765", - }, -}, { - summary: "Move (create) a directory with sticky bit", - options: fsutil.MoveOptions{ - Path: "foo", - Mode: fs.ModeDir | fs.ModeSticky | 0o775, - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/foo/": "dir 01775", - }, -}, { - summary: "Cannot move (create) a parent directory without MakeParents set", - options: fsutil.MoveOptions{ - Path: "foo/bar", - Mode: fs.ModeDir | 0o775, - }, - error: `mkdir /[^ ]*/foo/bar: no such file or directory`, -}, { - summary: "Do not override mode of existing directory", - options: fsutil.MoveOptions{ - Path: "foo", - Mode: fs.ModeDir | 0o775, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.Mkdir(filepath.Join(dir, "dst/foo/"), fs.ModeDir|0o765), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - // mode is not updated. - "/dst/foo/": "dir 0765", - }, -}, { - summary: "Moving to an existing file overrides the original mode", - options: fsutil.MoveOptions{ - Path: "foo", - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/foo"), []byte("data"), 0o644), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "dst/foo"), []byte("data"), 0o666), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/foo": "file 0644 3a6eb079", - }, -}, { - summary: "Move a hard link", - options: fsutil.MoveOptions{ - Path: "hardlink", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) - c.Assert(os.Link(filepath.Join(dir, "src/file"), filepath.Join(dir, "src/hardlink")), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/src/file": "file 0644 3a6eb079 <1>", - "/dst/hardlink": "file 0644 3a6eb079 <1>", - }, -}, { - summary: "No error if hard link already exists", - options: fsutil.MoveOptions{ - Path: "hardlink", - Mode: 0o644, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "dst/foo"), []byte("data"), 0o644), IsNil) - c.Assert(os.Link(filepath.Join(dir, "src/file"), filepath.Join(dir, "src/hardlink")), IsNil) - c.Assert(os.Link(filepath.Join(dir, "dst/foo"), filepath.Join(dir, "dst/hardlink")), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/src/file": "file 0644 3a6eb079 <1>", - "/dst/foo": "file 0644 3a6eb079", - "/dst/hardlink": "file 0644 3a6eb079 <1>", - }, -}, { - summary: "Override a symlink", - options: fsutil.MoveOptions{ - Path: "foo", - Mode: 0o666 | fs.ModeSymlink, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/baz"), []byte("data"), 0o666), IsNil) - c.Assert(os.Symlink("baz", filepath.Join(dir, "src/foo")), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "dst/bar"), []byte("data"), 0o644), IsNil) - c.Assert(os.Symlink("bar", filepath.Join(dir, "dst/foo")), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/src/baz": "file 0666 3a6eb079", - "/dst/": "dir 0755", - "/dst/bar": "file 0644 3a6eb079", - "/dst/foo": "symlink baz", - }, -}, { - summary: "Cannot move file from outside of Root", - options: fsutil.MoveOptions{ - SrcRoot: "/rootsrc", - DstRoot: "/rootdst", - Path: "../file", - Mode: 0o666, - }, - error: `cannot handle path /file outside of root /rootsrc`, -}, { - summary: "Path with ./ component", - options: fsutil.MoveOptions{ - Path: "./file", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/file": "file 0644 3a6eb079", - }, -}, { - summary: "Path with ../ component normalizes correctly", - options: fsutil.MoveOptions{ - Path: "foo/../bar", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/bar"), []byte("data"), 0o644), IsNil) - }, - result: map[string]string{ - "/src/": "dir 0755", - "/dst/": "dir 0755", - "/dst/bar": "file 0644 3a6eb079", - }, -}, { - summary: "Cannot move to a path where parent is a file", - options: fsutil.MoveOptions{ - Path: "file/subpath", - Mode: 0o644, - MakeParents: true, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/file"), []byte("data"), 0o644), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "dst/file"), []byte("data"), 0o644), IsNil) - }, - error: `mkdir .*/dst/file: not a directory`, -}, { - summary: "Cannot move a file to overwrite a directory", - options: fsutil.MoveOptions{ - Path: "target", - Mode: 0o644, - }, - hackopt: func(c *C, dir string, opts *fsutil.MoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "src/target"), []byte("data"), 0o644), IsNil) - c.Assert(os.Mkdir(filepath.Join(dir, "dst/target"), 0o755), IsNil) - }, - error: `rename .*/src/target .*/dst/target: file exists`, -}, { - summary: "Cannot move inexistent file", - options: fsutil.MoveOptions{ - Path: "file", - Mode: 0o666, - }, - error: `rename .*/src/file .*/dst/file: no such file or directory`, -}, { - summary: "Cannot move file to outside of Root", - options: fsutil.MoveOptions{ - SrcRoot: "/rootsrc", - DstRoot: "/rootdst", - Path: "../rootsrc/file", - Mode: 0o666, - }, - error: `cannot handle path /rootsrc/file outside of root /rootdst`, -}} - -func (s *S) TestMove(c *C) { - oldUmask := syscall.Umask(0) - defer func() { - syscall.Umask(oldUmask) - }() - - for _, test := range moveTests { - c.Logf("Test: %s", test.summary) - if test.result == nil { - // Empty map for no files moved. - test.result = make(map[string]string) - } - c.Logf("Options: %v", test.options) - dir := c.MkDir() - options := test.options - if options.SrcRoot == "" { - srcRoot := filepath.Join(dir, "src") - c.Assert(os.Mkdir(srcRoot, 0o755), IsNil) - options.SrcRoot = srcRoot - } - if options.DstRoot == "" { - dstRoot := filepath.Join(dir, "dst") - c.Assert(os.Mkdir(dstRoot, 0o755), IsNil) - options.DstRoot = dstRoot - } - if test.hackopt != nil { - test.hackopt(c, dir, &options) - } - err := fsutil.Move(&options) - - if test.error != "" { - c.Assert(err, ErrorMatches, test.error) - continue - } - - c.Assert(err, IsNil) - c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) - } -} - -func (s *S) TestMoveEmptyRoot(c *C) { - options := &fsutil.MoveOptions{SrcRoot: "", DstRoot: "foo/"} - err := fsutil.Move(options) - c.Assert(err, ErrorMatches, "internal error: MoveOptions.SrcRoot is unset") - options = &fsutil.MoveOptions{SrcRoot: "foo/", DstRoot: ""} - err = fsutil.Move(options) - c.Assert(err, ErrorMatches, "internal error: MoveOptions.DstRoot is unset") -} diff --git a/internal/fsutil/remove.go b/internal/fsutil/remove.go deleted file mode 100644 index 4f7cb865..00000000 --- a/internal/fsutil/remove.go +++ /dev/null @@ -1,50 +0,0 @@ -package fsutil - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "syscall" -) - -type RemoveOptions struct { - Root string - // Path is relative to Root. - Path string -} - -// Remove removes a filesystem entry according to the provided options. -// Non-empty directories are not removed. -// -// Remove can return errors from the os and syscall packages. -func Remove(options *RemoveOptions) error { - options, err := getValidRemoveOptions(options) - if err != nil { - return err - } - path, err := absPath(options.Root, options.Path) - if err != nil { - return err - } - err = os.Remove(path) - if err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENOTEMPTY) { - return err - } - return nil -} - -func getValidRemoveOptions(options *RemoveOptions) (*RemoveOptions, error) { - optsCopy := *options - o := &optsCopy - if o.Root == "" { - return nil, fmt.Errorf("internal error: RemoveOptions.Root is unset") - } - if o.Path == "" { - return nil, fmt.Errorf("internal error: RemoveOptions.Path is unset") - } - if o.Root != "/" { - o.Root = filepath.Clean(o.Root) + "/" - } - return o, nil -} diff --git a/internal/fsutil/remove_test.go b/internal/fsutil/remove_test.go deleted file mode 100644 index 4b744167..00000000 --- a/internal/fsutil/remove_test.go +++ /dev/null @@ -1,145 +0,0 @@ -package fsutil_test - -import ( - "os" - "path/filepath" - "syscall" - - . "gopkg.in/check.v1" - - "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/testutil" -) - -type removeTest struct { - summary string - options fsutil.RemoveOptions - hackopt func(c *C, dir string, opts *fsutil.RemoveOptions) - result map[string]string - error string -} - -var removeTests = []removeTest{{ - summary: "Remove a file", - options: fsutil.RemoveOptions{ - Path: "file", - }, - hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644), IsNil) - }, - result: map[string]string{}, -}, { - summary: "Remove a non-existent file", - options: fsutil.RemoveOptions{ - Path: "file", - }, - result: map[string]string{}, -}, { - summary: "Remove a non-existent directory", - options: fsutil.RemoveOptions{ - Path: "foo/", - }, - result: map[string]string{}, -}, { - summary: "Remove an empty directory", - options: fsutil.RemoveOptions{ - Path: "foo/bar", - }, - hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { - c.Assert(os.MkdirAll(filepath.Join(dir, "foo/bar"), 0o755), IsNil) - }, - result: map[string]string{ - "/foo/": "dir 0755", - }, -}, { - summary: "Do not remove non-empty directory", - options: fsutil.RemoveOptions{ - Path: "foo/", - }, - hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { - c.Assert(os.MkdirAll(filepath.Join(dir, "foo"), 0o755), IsNil) - c.Assert(os.WriteFile(filepath.Join(dir, "foo/file"), []byte("data"), 0o644), IsNil) - }, - result: map[string]string{ - "/foo/": "dir 0755", - "/foo/file": "file 0644 3a6eb079", - }, -}, { - summary: "Remove a symlink and not the target", - options: fsutil.RemoveOptions{ - Path: "bar", - }, - hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "foo"), []byte("data"), 0o644), IsNil) - c.Assert(os.Symlink("foo", filepath.Join(dir, "bar")), IsNil) - }, - result: map[string]string{ - "/foo": "file 0644 3a6eb079", - }, -}, { - summary: "Remove a hard link", - options: fsutil.RemoveOptions{ - Path: "hardlink1", - }, - hackopt: func(c *C, dir string, opts *fsutil.RemoveOptions) { - c.Assert(os.WriteFile(filepath.Join(dir, "file"), []byte("data"), 0o644), IsNil) - c.Assert(os.Link(filepath.Join(dir, "file"), filepath.Join(dir, "hardlink1")), IsNil) - c.Assert(os.Link(filepath.Join(dir, "file"), filepath.Join(dir, "hardlink2")), IsNil) - }, - result: map[string]string{ - "/file": "file 0644 3a6eb079 <1>", - "/hardlink2": "file 0644 3a6eb079 <1>", - }, -}, { - summary: "Cannot remove file outside of Root", - options: fsutil.RemoveOptions{ - Root: "/root", - Path: "../file", - }, - error: `cannot handle path /file outside of root /root/`, -}} - -func (s *S) TestRemove(c *C) { - oldUmask := syscall.Umask(0) - defer func() { - syscall.Umask(oldUmask) - }() - - for _, test := range removeTests { - c.Logf("Test: %s", test.summary) - if test.result == nil { - // Empty map for no files left. - test.result = make(map[string]string) - } - c.Logf("Options: %v", test.options) - dir := c.MkDir() - options := test.options - if options.Root == "" { - options.Root = dir - } - if test.hackopt != nil { - test.hackopt(c, dir, &options) - } - err := fsutil.Remove(&options) - - if test.error != "" { - c.Assert(err, ErrorMatches, test.error) - continue - } - - c.Assert(err, IsNil) - c.Assert(testutil.TreeDump(dir), DeepEquals, test.result) - } -} - -func (s *S) TestRemoveEmptyRoot(c *C) { - options := &fsutil.RemoveOptions{Root: ""} - err := fsutil.Remove(options) - c.Assert(err, ErrorMatches, "internal error: RemoveOptions.Root is unset") -} - -func (s *S) TestRemoveEmptyPath(c *C) { - options := &fsutil.RemoveOptions{Root: "foo/"} - err := fsutil.Remove(options) - c.Assert(err, ErrorMatches, "internal error: RemoveOptions.Path is unset") -} diff --git a/internal/manifestutil/manifestutil_test.go b/internal/manifestutil/manifestutil_test.go index 216c68e2..f096e15b 100644 --- a/internal/manifestutil/manifestutil_test.go +++ b/internal/manifestutil/manifestutil_test.go @@ -150,6 +150,7 @@ var findPathsInReleaseTests = []struct { }, }, }, + expected: nil, }, { summary: "Multiple packages with multiple slices", release: &setup.Release{ @@ -203,6 +204,7 @@ var findPathsInReleaseTests = []struct { release: &setup.Release{ Packages: map[string]*setup.Package{}, }, + expected: nil, }} func (s *S) TestFindPathsInRelease(c *C) { diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 8322dee3..52969b28 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -5,6 +5,7 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "io/fs" @@ -96,13 +97,12 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - var originalTargetDir string + originalTargetDir := targetDir if options.Manifest != nil { tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") if err != nil { return fmt.Errorf("cannot create temporary working directory: %w", err) } - originalTargetDir = targetDir targetDir = tmpWorkDir defer func() { os.RemoveAll(tmpWorkDir) @@ -366,15 +366,14 @@ func Run(options *RunOptions) error { return err } - err = generateManifests(targetDir, options.Selection, report, pkgInfos) - if err != nil { - return err - } - if options.Manifest != nil { - return upgrade(originalTargetDir, targetDir, report, options.Manifest) + err = upgrade(originalTargetDir, targetDir, report, options.Manifest) + if err != nil { + return err + } } - return nil + + return generateManifests(originalTargetDir, options.Selection, report, pkgInfos) } // upgrade upgrades content in targetDir with content in tempDir. @@ -383,13 +382,23 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes paths := slices.Sorted(maps.Keys(report.Entries)) for _, path := range paths { entry := report.Entries[path] - err := fsutil.Move(&fsutil.MoveOptions{ - SrcRoot: tempDir, - DstRoot: targetDir, - Path: path, - Mode: entry.Mode, - MakeParents: true, - }) + srcPath := filepath.Clean(filepath.Join(tempDir, path)) + dstPath := filepath.Clean(filepath.Join(targetDir, path)) + if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + return err + } + var err error + switch entry.Mode & fs.ModeType { + case 0, fs.ModeSymlink: + err = os.Rename(srcPath, dstPath) + case fs.ModeDir: + mkdirErr := os.Mkdir(dstPath, entry.Mode) + if !os.IsExist(mkdirErr) { + err = mkdirErr + } + default: + err = fmt.Errorf("unsupported file type: %s", path) + } if err != nil { return err } @@ -409,14 +418,13 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes } sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) for _, relPath := range missingPaths { - err := fsutil.Remove(&fsutil.RemoveOptions{ - Root: targetDir, - Path: relPath, - }) - if err != nil { + path := filepath.Clean(filepath.Join(targetDir, relPath)) + err = os.Remove(path) + if err != nil && !os.IsNotExist(err) && !errors.Is(err, syscall.ENOTEMPTY) { return err } } + report.Root = targetDir return nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 17240857..041dc18b 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1982,7 +1982,7 @@ var slicerTests = []slicerTest{{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, }, { - summary: "Recut removes obsolete paths when selection shrinks", + summary: "Upgrade removes obsolete paths when selection shrinks", slices: []setup.SliceKey{{"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -2034,7 +2034,7 @@ var slicerTests = []slicerTest{{ "/bar": "file 0644 d98cf53e {test-package_slice2}", }, }, { - summary: "Recut restores modified content and mode", + summary: "Upgrade restores modified content and mode", slices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -2049,11 +2049,11 @@ var slicerTests = []slicerTest{{ pkg := release.Packages["test-package"] slice1 := pkg.Slices["slice1"] manifestSlice := pkg.Slices["manifest"] - writeFile(c, opts.TargetDir, "/file", []byte("data1"), 0o644) + filename := "file" report, err := manifestutil.NewReport(opts.TargetDir) c.Assert(err, IsNil) err = report.Add(slice1, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/file"), + Path: filepath.Join(report.Root, filename), Mode: 0o644, SHA256: "5b41362b", Size: 5, @@ -2065,7 +2065,7 @@ var slicerTests = []slicerTest{{ }) c.Assert(err, IsNil) writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, manifestSlice}, report) - modifiedPath := filepath.Join(opts.TargetDir, "file") + modifiedPath := filepath.Join(opts.TargetDir, filename) err = os.WriteFile(modifiedPath, []byte("data2"), 0o700) c.Assert(err, IsNil) opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) @@ -2077,7 +2077,7 @@ var slicerTests = []slicerTest{{ "/file": "file 0644 5b41362b {test-package_slice1}", }, }, { - summary: "Recut keeps untracked files", + summary: "Upgrade keeps untracked files", slices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` @@ -2122,6 +2122,277 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/file": "file 0644 5b41362b {test-package_slice1}", }, +}, { + summary: "Upgrade creates parent directory", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /foo/bar: {text: data} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/foo/": "dir 0755", + "/foo/bar": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/foo/bar": "file 0644 3a6eb079 {test-package_myslice}", + }, +}, { + summary: "Upgrade creates symlink", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /baz: {text: data} + /foo: {symlink: baz} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/baz": "file 0644 3a6eb079", + "/foo": "symlink baz", + }, + manifestPaths: map[string]string{ + "/baz": "file 0644 3a6eb079 {test-package_myslice}", + "/foo": "symlink baz {test-package_myslice}", + }, +}, { + summary: "Upgrade does not override existing directory mode", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/: {make: true, mode: 0775} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + err := os.Mkdir(filepath.Join(opts.TargetDir, "dir"), 0o700) + c.Assert(err, IsNil) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/dir/": "dir 0700", + }, + manifestPaths: map[string]string{ + "/dir/": "dir 0775 {test-package_myslice}", + }, +}, { + summary: "Upgrade overwrites existing file mode", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /foo: {text: data} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + writeFile(c, opts.TargetDir, "/foo", []byte("data"), 0o600) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/foo": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/foo": "file 0644 3a6eb079 {test-package_myslice}", + }, +}, { + summary: "Upgrade overwrites existing symlink", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /baz: {text: data} + /foo: {symlink: baz} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + writeFile(c, opts.TargetDir, "/bar", []byte("data"), 0o644) + c.Assert(os.Symlink("bar", filepath.Join(opts.TargetDir, "foo")), IsNil) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/bar": "file 0644 3a6eb079", + "/baz": "file 0644 3a6eb079", + "/foo": "symlink baz", + }, + manifestPaths: map[string]string{ + "/baz": "file 0644 3a6eb079 {test-package_myslice}", + "/foo": "symlink baz {test-package_myslice}", + }, +}, { + summary: "Upgrade fails when parent is a file", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /dir/file: {text: data} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + writeFile(c, opts.TargetDir, "/dir", []byte("data"), 0o644) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + error: `mkdir .*: not a directory`, +}, { + summary: "Upgrade fails when target path is a directory", + slices: []setup.SliceKey{{"test-package", "myslice"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + myslice: + contents: + /target: {text: data} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "target"), 0o755), IsNil) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + error: `rename .*: (file exists|is a directory)`, +}, { + summary: "Upgrade removes obsolete empty directory", + slices: []setup.SliceKey{{"test-package", "new"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + old: + contents: + /old-dir/: {make: true} + new: + contents: + /new-file: {text: data1} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "old-dir"), 0o755), IsNil) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + oldSlice := release.Packages["test-package"].Slices["old"] + err := report.Add(oldSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/old-dir"), + Mode: fs.ModeDir | 0o755, + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/new-file": "file 0644 5b41362b", + }, + manifestPaths: map[string]string{ + "/new-file": "file 0644 5b41362b {test-package_new}", + }, +}, { + summary: "Upgrade keeps obsolete non-empty directory", + slices: []setup.SliceKey{{"test-package", "new"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + old: + contents: + /old-dir/: {make: true} + new: + contents: + /new-file: {text: data1} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "old-dir"), 0o755), IsNil) + writeFile(c, opts.TargetDir, "/old-dir/file", []byte("data"), 0o644) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + oldSlice := release.Packages["test-package"].Slices["old"] + err := report.Add(oldSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/old-dir"), + Mode: fs.ModeDir | 0o755, + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/new-file": "file 0644 5b41362b", + "/old-dir/": "dir 0755", + "/old-dir/file": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/new-file": "file 0644 5b41362b {test-package_new}", + }, +}, { + summary: "Upgrade removes obsolete symlink only", + slices: []setup.SliceKey{{"test-package", "new"}}, + release: map[string]string{ + "slices/mydir/test-package.yaml": ` + package: test-package + slices: + old: + contents: + /link: {symlink: target} + new: + contents: + /new-file: {text: data1} + `, + }, + prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { + writeFile(c, opts.TargetDir, "/target", []byte("data"), 0o644) + c.Assert(os.Symlink("target", filepath.Join(opts.TargetDir, "link")), IsNil) + report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") + oldSlice := release.Packages["test-package"].Slices["old"] + err := report.Add(oldSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, "/link"), + Mode: fs.ModeSymlink, + Link: "target", + }) + c.Assert(err, IsNil) + writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) + opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + }, + filesystem: map[string]string{ + "/new-file": "file 0644 5b41362b", + "/target": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/new-file": "file 0644 5b41362b {test-package_new}", + }, }} func (s *S) TestRun(c *C) { @@ -2531,6 +2802,18 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { return mfest } +func newManifestReport(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath, pkgName string) (*manifestutil.Report, *setup.Slice) { + report, err := manifestutil.NewReport(opts.TargetDir) + c.Assert(err, IsNil) + manifestSlice := release.Packages[pkgName].Slices["manifest"] + err = report.Add(manifestSlice, &fsutil.Entry{ + Path: filepath.Join(report.Root, manifestPath), + Mode: 0o644, + }) + c.Assert(err, IsNil) + return report, manifestSlice +} + func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { mfestPath := filepath.Join(targetDir, manifestPath) err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index a01edcb9..2b3789bc 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -33,7 +33,7 @@ execute: | EXTRA_OPTIONS="--ignore=unmaintained " fi - # First cut generates manifest and installs slice-a + # First cut generates manifest and installs slice-a. chisel cut --release $chisel_release \ --root $rootfs_folder \ $EXTRA_OPTIONS \ From 7caf3badba9f1f82500296732f9f5db933fe50f7 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 6 Feb 2026 14:27:27 +0100 Subject: [PATCH 29/65] refactor: revert error message change --- internal/deb/extract_test.go | 2 +- internal/fsutil/create.go | 2 +- internal/fsutil/create_test.go | 4 ++-- internal/slicer/slicer_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/deb/extract_test.go b/internal/deb/extract_test.go index 30f77816..1ec0b8a5 100644 --- a/internal/deb/extract_test.go +++ b/internal/deb/extract_test.go @@ -485,7 +485,7 @@ var extractTests = []extractTest{{ }}, }, }, - error: `cannot extract from package "test-package": cannot handle path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, + error: `cannot extract from package "test-package": cannot create path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, }} func (s *S) TestExtract(c *C) { diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 59016145..1478e715 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -261,7 +261,7 @@ func getValidCreateOptions(options *CreateOptions) (*CreateOptions, error) { func absPath(root, relPath string) (string, error) { path := filepath.Clean(filepath.Join(root, relPath)) if !strings.HasPrefix(path, root) { - return "", fmt.Errorf("cannot handle path %s outside of root %s", path, root) + return "", fmt.Errorf("cannot create path %s outside of root %s", path, root) } return path, nil } diff --git a/internal/fsutil/create_test.go b/internal/fsutil/create_test.go index 370568c0..5e908fcc 100644 --- a/internal/fsutil/create_test.go +++ b/internal/fsutil/create_test.go @@ -279,7 +279,7 @@ var createTests = []createTest{{ Mode: 0666, Data: bytes.NewBufferString("hijacking system file"), }, - error: `cannot handle path /file outside of root /root/`, + error: `cannot create path /file outside of root /root/`, }, { summary: "Hardlink cannot escape Root", options: fsutil.CreateOptions{ @@ -410,7 +410,7 @@ var createWriterTests = []createWriterTest{{ Mode: 0644, MakeParents: true, }, - error: `cannot handle path /file outside of root /root/`, + error: `cannot create path /file outside of root /root/`, }} func (s *S) TestCreateWriter(c *C) { diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 041dc18b..33b70d54 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -1809,7 +1809,7 @@ var slicerTests = []slicerTest{{ /**: `, }, - error: `cannot extract from package "test-package": cannot handle path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, + error: `cannot extract from package "test-package": cannot create path /[a-z0-9\-\/]*/file outside of root /[a-z0-9\-\/]*`, }, { summary: "Extract conflicting paths with prefer from proper package", slices: []setup.SliceKey{ From a259d1d581fcd425520d474db5902e626c6e202d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 08:32:26 +0100 Subject: [PATCH 30/65] fix: gate recut feature behind env var --- cmd/chisel/cmd_cut.go | 30 +++++++++++++++++------------- tests/recut/task.yaml | 1 + 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index b0dd1eec..1d0abfd9 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "slices" "time" @@ -74,24 +75,27 @@ func (cmd *cmdCut) Execute(args []string) error { } } - mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) - if err != nil { - return err - } - if mfest != nil { - err = mfest.IterateSlices("", func(slice *manifest.Slice) error { - sk, err := setup.ParseSliceKey(slice.Name) + var mfest *manifest.Manifest + // TODO: Remove this gating once the final upgrading strategy is in place. + if os.Getenv("CHISEL_RECUT") != "" { + mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + err = mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) if err != nil { return err } - sliceKeys = append(sliceKeys, sk) - return nil - }) - if err != nil { - return err } } - selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index 2b3789bc..a1dc3c8f 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -34,6 +34,7 @@ execute: | fi # First cut generates manifest and installs slice-a. + export CHISEL_RECUT=1 chisel cut --release $chisel_release \ --root $rootfs_folder \ $EXTRA_OPTIONS \ From fad2cb7ab0e9ee00500e2a469a3c66d63eb57040 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 09:20:52 +0100 Subject: [PATCH 31/65] refactor: split install from Run --- internal/slicer/slicer.go | 58 +++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 52969b28..815a7213 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -108,15 +108,34 @@ func Run(options *RunOptions) error { os.RemoveAll(tmpWorkDir) }() } + optsCopy := *options + installOpts := &optsCopy + installOpts.TargetDir = targetDir - pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + report, pkgInfos, err := install(installOpts) if err != nil { return err } + if options.Manifest != nil { + err = upgrade(originalTargetDir, targetDir, report, options.Manifest) + if err != nil { + return err + } + } + + return generateManifests(originalTargetDir, options.Selection, report, pkgInfos) +} + +func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, error) { + pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) + if err != nil { + return nil, nil, err + } + prefers, err := options.Selection.Prefers() if err != nil { - return err + return nil, nil, err } // Build information to process the selection. @@ -173,7 +192,7 @@ func Run(options *RunOptions) error { } reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) if err != nil { - return err + return nil, nil, err } defer reader.Close() packages[slice.Package] = reader @@ -184,9 +203,9 @@ func Run(options *RunOptions) error { // listed as until: mutate in all the slices that reference them. knownPaths := map[string]pathData{} addKnownPath(knownPaths, "/", pathData{}) - report, err := manifestutil.NewReport(targetDir) + report, err := manifestutil.NewReport(options.TargetDir) if err != nil { - return fmt.Errorf("internal error: cannot create report: %w", err) + return nil, nil, fmt.Errorf("internal error: cannot create report: %w", err) } // Record directories which are created but where not listed in the slice @@ -202,7 +221,7 @@ func Run(options *RunOptions) error { return err } - relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, targetDir)) + relPath := filepath.Clean("/" + strings.TrimPrefix(o.Path, options.TargetDir)) if o.Mode.IsDir() { relPath = relPath + "/" } @@ -260,13 +279,13 @@ func Run(options *RunOptions) error { err := deb.Extract(reader, &deb.ExtractOptions{ Package: slice.Package, Extract: extract[slice.Package], - TargetDir: targetDir, + TargetDir: options.TargetDir, Create: create, }) reader.Close() packages[slice.Package] = nil if err != nil { - return err + return nil, nil, err } } @@ -322,9 +341,9 @@ func Run(options *RunOptions) error { mutable: pathInfo.Mutable, } addKnownPath(knownPaths, relPath, data) - entry, err := createFile(targetDir, relPath, pathInfo) + entry, err := createFile(options.TargetDir, relPath, pathInfo) if err != nil { - return err + return nil, nil, err } // Do not add paths with "until: mutate". @@ -332,7 +351,7 @@ func Run(options *RunOptions) error { for _, slice := range slices { err = report.Add(slice, entry) if err != nil { - return err + return nil, nil, err } } } @@ -342,7 +361,7 @@ func Run(options *RunOptions) error { // dependencies must run before dependents. checker := contentChecker{knownPaths} content := &scripts.ContentValue{ - RootDir: targetDir, + RootDir: options.TargetDir, CheckWrite: checker.checkMutable, CheckRead: checker.checkKnown, OnWrite: report.Mutate, @@ -357,23 +376,16 @@ func Run(options *RunOptions) error { } err := scripts.Run(&opts) if err != nil { - return fmt.Errorf("slice %s: %w", slice, err) + return nil, nil, fmt.Errorf("slice %s: %w", slice, err) } } - err = removeAfterMutate(targetDir, knownPaths) + err = removeAfterMutate(options.TargetDir, knownPaths) if err != nil { - return err - } - - if options.Manifest != nil { - err = upgrade(originalTargetDir, targetDir, report, options.Manifest) - if err != nil { - return err - } + return nil, nil, err } - return generateManifests(originalTargetDir, options.Selection, report, pkgInfos) + return report, pkgInfos, nil } // upgrade upgrades content in targetDir with content in tempDir. From d09595393db0c4a4112ecb59db08e21d511db10f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 10:19:16 +0100 Subject: [PATCH 32/65] tests: simplify recut spread test --- tests/recut/task.yaml | 69 ++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index a1dc3c8f..515f36fa 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -1,57 +1,32 @@ -summary: Recut relies on manifest to update existing content +summary: Recut relies on manifest to upgrade existing content + +variants: + - noble + +environment: + ROOTFS: rootfs execute: | - rootfs_folder=rootfs_${RELEASE} - mkdir -p $rootfs_folder - - chisel_release="./release_${RELEASE}" - mkdir -p ${chisel_release}/slices - - ref_chisel_release="ref-chisel-release_${RELEASE}" - git clone --depth=1 -b ${OS}-${RELEASE} \ - https://github.com/canonical/chisel-releases $ref_chisel_release - - cp ${ref_chisel_release}/chisel.yaml ${chisel_release}/chisel.yaml - - cat >${chisel_release}/slices/base-files.yaml <<'EOF' - package: base-files - - slices: - slice-a: - contents: - /etc/debian_version: - slice-b: - contents: - /etc/issue: - manifest: - contents: - /chisel/**: {generate: manifest} - EOF - - EXTRA_OPTIONS="" - if [ "$RELEASE" = "23.10" -o "$RELEASE" = "20.04" ]; then - EXTRA_OPTIONS="--ignore=unmaintained " - fi + mkdir -p ${ROOTFS} - # First cut generates manifest and installs slice-a. + # TODO: remove this env var when final upgrade strategy is in place. export CHISEL_RECUT=1 - chisel cut --release $chisel_release \ - --root $rootfs_folder \ - $EXTRA_OPTIONS \ + # First cut generates manifest and installs slice-a. + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS} \ base-files_slice-a base-files_manifest - test -s ${rootfs_folder}/etc/debian_version - test -f ${rootfs_folder}/chisel/manifest.wall + test -s ${ROOTFS}/etc/debian_version + test -f ${ROOTFS}/chisel/manifest.wall - # Remove a file from the first slice - rm -f ${rootfs_folder}/etc/debian_version + # Remove a file installed from slice-a. + rm -f ${ROOTFS}/etc/debian_version - # Second cut only requests slice-b, slice-a should be restored via manifest - chisel cut --release $chisel_release \ - --root $rootfs_folder \ - $EXTRA_OPTIONS \ + # Second cut, only requesting slice-b. slice-a should be restored via manifest. + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS} \ base-files_slice-b - test -s ${rootfs_folder}/etc/debian_version - test -s ${rootfs_folder}/etc/issue - test -f ${rootfs_folder}/chisel/manifest.wall + test -s ${ROOTFS}/etc/debian_version + test -s ${ROOTFS}/etc/issue + test -f ${ROOTFS}/chisel/manifest.wall From d0b70cda6fabf07c0c9a288197ab941508f93755 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 10:48:40 +0100 Subject: [PATCH 33/65] tests: rework recut spread tests --- tests/recut/chisel-releases/chisel.yaml | 49 +++++++++++++++++++ .../chisel-releases/slices/base-files.yaml | 13 +++++ tests/recut/task.yaml | 13 +++-- 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 tests/recut/chisel-releases/chisel.yaml create mode 100644 tests/recut/chisel-releases/slices/base-files.yaml diff --git a/tests/recut/chisel-releases/chisel.yaml b/tests/recut/chisel-releases/chisel.yaml new file mode 100644 index 00000000..c137ff40 --- /dev/null +++ b/tests/recut/chisel-releases/chisel.yaml @@ -0,0 +1,49 @@ +format: v2 + + +archives: + # archive.ubuntu.com/ubuntu/ (amd64, i386) + # ports.ubuntu.com/ubuntu-ports/ (other arch) + ubuntu: + priority: 10 + version: 24.04 + components: [main, universe] + suites: [noble, noble-security, noble-updates] + public-keys: [ubuntu-archive-key-2018] + + +public-keys: + # Ubuntu Archive Automatic Signing Key (2018) + # rsa4096/f6ecb3762474eda9d21b7022871920d1991bc93c 2018-09-17T15:01:46Z + ubuntu-archive-key-2018: + id: "871920D1991BC93C" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq + dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ + HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq + X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k + KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns + AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta + LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq + VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd + yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv + Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow + DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB + tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm + dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI + AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH + beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG + g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa + Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy + q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF + kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0 + LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr + BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5 + KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA + QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp + SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+ + uOgcXny1UlwtCUzlrSaP + =9AdM + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/recut/chisel-releases/slices/base-files.yaml b/tests/recut/chisel-releases/slices/base-files.yaml new file mode 100644 index 00000000..84c30016 --- /dev/null +++ b/tests/recut/chisel-releases/slices/base-files.yaml @@ -0,0 +1,13 @@ +package: base-files +slices: + slice-a: + contents: + /etc/debian_version: + /etc/foo: + text: bar + slice-b: + contents: + /etc/issue: + manifest: + contents: + /chisel/**: {generate: manifest} diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index 515f36fa..bdffcc6e 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -7,6 +7,10 @@ environment: ROOTFS: rootfs execute: | + # Install yq. + wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ + chmod +x /usr/bin/yq + mkdir -p ${ROOTFS} # TODO: remove this env var when final upgrade strategy is in place. @@ -17,16 +21,19 @@ execute: | base-files_slice-a base-files_manifest test -s ${ROOTFS}/etc/debian_version + test -s ${ROOTFS}/etc/foo + cat ${ROOTFS}/etc/foo | grep "bar" test -f ${ROOTFS}/chisel/manifest.wall - # Remove a file installed from slice-a. - rm -f ${ROOTFS}/etc/debian_version + # Update slice-a definition. + yq -i '.slices.slice-a.contents./etc/foo.text = "qux"' ./chisel-releases/slices/base-files.yaml - # Second cut, only requesting slice-b. slice-a should be restored via manifest. + # Second cut, only requesting slice-b. chisel cut --release ./chisel-releases/ \ --root ${ROOTFS} \ base-files_slice-b test -s ${ROOTFS}/etc/debian_version test -s ${ROOTFS}/etc/issue + cat ${ROOTFS}/etc/foo | grep "qux" test -f ${ROOTFS}/chisel/manifest.wall From 83e7af65598d8342cd96c446be3d72ad600b39bd Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 13:47:57 +0100 Subject: [PATCH 34/65] refactor: revert now useless changes --- internal/fsutil/create.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 1478e715..8c5b58e6 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -42,7 +42,7 @@ type Entry struct { // // Create can return errors from the os package. func Create(options *CreateOptions) (*Entry, error) { - o, err := getValidCreateOptions(options) + o, err := getValidOptions(options) if err != nil { return nil, err } @@ -124,7 +124,7 @@ func Create(options *CreateOptions) (*Entry, error) { // information recorded in Entry. The Hash and Size attributes are set on // calling Close() on the Writer. func CreateWriter(options *CreateOptions) (io.WriteCloser, *Entry, error) { - o, err := getValidCreateOptions(options) + o, err := getValidOptions(options) if err != nil { return nil, nil, err } @@ -245,7 +245,7 @@ func createHardLink(o *CreateOptions) error { return err } -func getValidCreateOptions(options *CreateOptions) (*CreateOptions, error) { +func getValidOptions(options *CreateOptions) (*CreateOptions, error) { optsCopy := *options o := &optsCopy if o.Root == "" { From d217c43c87a6f0f1850c41e2b49adf804b4cfaa1 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 14:52:48 +0100 Subject: [PATCH 35/65] refactor: simplify --- internal/manifestutil/manifestutil.go | 8 --- internal/slicer/slicer.go | 82 +++++++++++---------------- 2 files changed, 34 insertions(+), 56 deletions(-) diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 63a7be83..071ed326 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -359,11 +359,3 @@ func Validate(mfest *manifest.Manifest) (err error) { } return nil } - -// CompareSchemas compares two manifest schema strings. -func CompareSchemas(va, vb string) int { - if va == manifest.Schema && va == vb { - return 0 - } - return -1 -} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 815a7213..87998b31 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -628,9 +628,8 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return pkgArchive, nil } -// SelectValidManifest returns, if found, a valid manifest with the latest -// schema. Consistency with all other manifests with the same schema is verified -// so the selection is deterministic. +// SelectValidManifest returns, if found, a valid manifest. Consistency with +// other manifests is verified so the selection is deterministic. func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { targetDir = filepath.Clean(targetDir) if !filepath.IsAbs(targetDir) { @@ -645,57 +644,44 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma return nil, nil } - type manifestHash struct { - path string - hash string - } var selected *manifest.Manifest - schemaManifest := make(map[string]manifestHash) + var selectedHash string + var selectedPath string for _, mfestPath := range manifestPaths { - err := func() error { - mfestFullPath := path.Join(targetDir, mfestPath) - f, err := os.Open(mfestFullPath) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer f.Close() - r, err := zstd.NewReader(f) - if err != nil { - return err - } - defer r.Close() - mfest, err := manifest.Read(r) - if err != nil { - return err - } - err = manifestutil.Validate(mfest) - if err != nil { - return err - } - // Verify consistency with other manifests with the same schema. - h, err := contentHash(mfestFullPath) - if err != nil { - return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) - } - mfestHash := hex.EncodeToString(h) - refMfest, ok := schemaManifest[mfest.Schema()] - if !ok { - schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} - } else if refMfest.hash != mfestHash { - return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) - } - - if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { - selected = mfest + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + continue } - return nil - }() + return nil, err + } + defer f.Close() + r, err := zstd.NewReader(f) if err != nil { return nil, err } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + return nil, err + } + err = manifestutil.Validate(mfest) + if err != nil { + return nil, err + } + h, err := contentHash(mfestFullPath) + if err != nil { + return nil, fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + if selected == nil { + selected = mfest + selectedHash = mfestHash + selectedPath = mfestPath + } else if selectedHash != mfestHash { + return nil, fmt.Errorf("inconsistent manifests: %q and %q", selectedPath, mfestPath) + } } return selected, nil } From 8a7066bb3db54bb217c2752c81da8cea966d58d3 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 9 Feb 2026 14:53:58 +0100 Subject: [PATCH 36/65] refactor: more cleaning --- public/manifest/manifest.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 65b362d5..1e4809b8 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -68,10 +68,6 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return manifest, nil } -func (manifest *Manifest) Schema() string { - return manifest.db.Schema() -} - func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) } From 87d1496c63bc13c1bca528f950308b43fb8f920d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Tue, 10 Feb 2026 15:09:07 +0100 Subject: [PATCH 37/65] test: wip rework uprade tests --- internal/slicer/slicer_test.go | 952 +++++++++++++++++---------------- 1 file changed, 483 insertions(+), 469 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 33b70d54..a7c35f7e 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -15,7 +15,6 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" - "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" @@ -35,7 +34,6 @@ type slicerTest struct { pkgs []*testutil.TestPackage slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) - prefill func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) filesystem map[string]string manifestPaths map[string]string manifestPkgs map[string]string @@ -1981,249 +1979,384 @@ var slicerTests = []slicerTest{{ manifestPaths: map[string]string{ "/dir/file": "file 0644 cc55e2ec {test-package_third}", }, -}, { - summary: "Upgrade removes obsolete paths when selection shrinks", - slices: []setup.SliceKey{{"test-package", "slice2"}}, +}} + +func (s *S) TestRun(c *C) { + // Run tests for "archives" field in "v1" format. + runSlicerTests(s, c, slicerTests) + + // Run tests for "v2-archives" field in "v1" format. + v2ArchiveTests := make([]slicerTest, 0, len(slicerTests)) + for _, t := range slicerTests { + m := make(map[string]string) + for k, v := range t.release { + if !strings.Contains(v, "v2-archives:") { + v = strings.Replace(v, "archives:", "v2-archives:", -1) + } + m[k] = v + } + t.release = m + v2ArchiveTests = append(v2ArchiveTests, t) + } + runSlicerTests(s, c, v2ArchiveTests) + + // Run tests for "v2" format. + v2FormatTests := make([]slicerTest, 0, len(slicerTests)) + for _, t := range slicerTests { + m := make(map[string]string) + for k, v := range t.release { + if strings.Contains(v, "format: v1") && + !strings.Contains(v, "v2-archives:") && + !strings.Contains(v, "default: true") { + v = strings.Replace(v, "format: v1", "format: v2", -1) + } + m[k] = v + } + t.release = m + v2FormatTests = append(v2FormatTests, t) + } + runSlicerTests(s, c, v2FormatTests) +} + +func runSlicerTests(s *S, c *C, tests []slicerTest) { + for _, test := range tests { + for _, testSlices := range testutil.Permutations(test.slices) { + const logMarker = "---log-marker---" + c.Logf(logMarker) + c.Logf("Summary: %s", test.summary) + + if _, ok := test.release["chisel.yaml"]; !ok { + test.release["chisel.yaml"] = testutil.DefaultChiselYaml + } + if test.pkgs == nil { + test.pkgs = []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }} + } + for _, pkg := range test.pkgs { + // We need to set these fields for manifest validation. + if pkg.Arch == "" { + pkg.Arch = "arch" + } + if pkg.Hash == "" { + pkg.Hash = "hash" + } + if pkg.Version == "" { + pkg.Version = "version" + } + } + + releaseDir := c.MkDir() + for path, data := range test.release { + fpath := filepath.Join(releaseDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(releaseDir) + c.Assert(err, IsNil) + + // Create a manifest slice and add it to the selection. + manifestPackage := test.slices[0].Package + manifestPath := "/chisel-data/manifest.wall" + release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ + Package: manifestPackage, + Name: "manifest", + Essential: nil, + Contents: map[string]setup.PathInfo{ + "/chisel-data/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + Scripts: setup.SliceScripts{}, + } + testSlices = append(testSlices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", + }) + + selection, err := setup.Select(release, testSlices, test.arch) + c.Assert(err, IsNil) + + archives := map[string]archive.Archive{} + for name, setupArchive := range release.Archives { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { + pkgs[pkg.Name] = pkg + } + } + archive := &testutil.TestArchive{ + Opts: archive.Options{ + Label: setupArchive.Name, + Version: setupArchive.Version, + Suites: setupArchive.Suites, + Components: setupArchive.Components, + Pro: setupArchive.Pro, + Arch: test.arch, + }, + Packages: pkgs, + } + archives[name] = archive + } + + options := slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: c.MkDir(), + } + if test.hackopt != nil { + test.hackopt(c, &options) + } + err = slicer.Run(&options) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + + if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil && test.logOutput == "" { + continue + } + mfest := readManifest(c, options.TargetDir, manifestPath) + + // Assert state of final filesystem. + if test.filesystem != nil { + filesystem := testutil.TreeDump(options.TargetDir) + c.Assert(filesystem["/chisel-data/"], Not(HasLen), 0) + c.Assert(filesystem[manifestPath], Not(HasLen), 0) + delete(filesystem, "/chisel-data/") + delete(filesystem, manifestPath) + c.Assert(filesystem, DeepEquals, test.filesystem) + } + + // Assert state of the files recorded in the manifest. + if test.manifestPaths != nil { + pathsDump, err := treeDumpManifestPaths(mfest) + c.Assert(err, IsNil) + c.Assert(pathsDump[manifestPath], Not(HasLen), 0) + delete(pathsDump, manifestPath) + c.Assert(pathsDump, DeepEquals, test.manifestPaths) + } + + // Assert state of the packages recorded in the manifest. + if test.manifestPkgs != nil { + pkgsDump, err := dumpManifestPkgs(mfest) + c.Assert(err, IsNil) + c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) + } + + // Find the log output of this test by trimming the suite output + // until we find the last occurrence of the summary. + testLogs := strings.Split(c.GetTestLog(), logMarker) + logOutput := testLogs[len(testLogs)-1] + + // Assert log output. + if test.logOutput != "" { + c.Assert(logOutput, Matches, test.logOutput) + } else { + // No warnings emitted. + c.Assert(logOutput, Not(Matches), "(?s).*Warning.*") + } + } + } +} + +type slicerRecutTest struct { + summary string + arch string + release map[string]string + pkgs []*testutil.TestPackage + cutSlices []setup.SliceKey + recutSlices []setup.SliceKey + hackopt func(c *C, opts *slicer.RunOptions) + hackRecutOpt func(c *C, opts *slicer.RunOptions) + prefill func(c *C, targetDir string) + alterFilesystem func(c *C, targetDir string) + // Modifies the filesystem built after the first execution and before the + // second one. + filesystem map[string]string + manifestPaths map[string]string + manifestPkgs map[string]string + logOutput string + error string +} + +var slicerRecutTests = []slicerRecutTest{{ + summary: "Basic upgrade", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: slice1: contents: - /foo: {text: data1} + /dir/file: + /dir/file-copy: {copy: /dir/file} + /other-dir/file: {symlink: ../dir/file} slice2: contents: - /bar: {text: data2} + /dir/other-file: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - pkg := release.Packages["test-package"] - slice1 := pkg.Slices["slice1"] - slice2 := pkg.Slices["slice2"] - manifestSlice := pkg.Slices["manifest"] - writeFile(c, opts.TargetDir, "/foo", []byte("data1"), 0o644) - writeFile(c, opts.TargetDir, "/bar", []byte("data2"), 0o644) - report, err := manifestutil.NewReport(opts.TargetDir) - c.Assert(err, IsNil) - err = report.Add(slice1, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/foo"), - Mode: 0o644, - SHA256: "5b41362b", - Size: 5, - }) - c.Assert(err, IsNil) - err = report.Add(slice2, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/bar"), - Mode: 0o644, - SHA256: "d98cf53e", - Size: 5, - }) - c.Assert(err, IsNil) - err = report.Add(manifestSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, manifestPath), - Mode: 0o644, - }) - c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, slice2, manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, filesystem: map[string]string{ - "/bar": "file 0644 d98cf53e", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/file-copy": "file 0644 cc55e2ec", + "/dir/other-file": "file 0644 63d5dd49", + "/other-dir/": "dir 0755", + "/other-dir/file": "symlink ../dir/file", }, manifestPaths: map[string]string{ - "/bar": "file 0644 d98cf53e {test-package_slice2}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/file-copy": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", + "/other-dir/file": "symlink ../dir/file {test-package_slice1}", }, }, { - summary: "Upgrade restores modified content and mode", - slices: []setup.SliceKey{{"test-package", "slice1"}}, + summary: "Upgrade removes obsolete paths when selection shrinks", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: slice1: contents: - /file: {text: data1} + /dir/file: + slice2: + contents: + /dir/other-file: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - pkg := release.Packages["test-package"] - slice1 := pkg.Slices["slice1"] - manifestSlice := pkg.Slices["manifest"] - filename := "file" - report, err := manifestutil.NewReport(opts.TargetDir) - c.Assert(err, IsNil) - err = report.Add(slice1, &fsutil.Entry{ - Path: filepath.Join(report.Root, filename), - Mode: 0o644, - SHA256: "5b41362b", - Size: 5, - }) - c.Assert(err, IsNil) - err = report.Add(manifestSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, manifestPath), - Mode: 0o644, - }) - c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, manifestSlice}, report) - modifiedPath := filepath.Join(opts.TargetDir, filename) - err = os.WriteFile(modifiedPath, []byte("data2"), 0o700) - c.Assert(err, IsNil) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, filesystem: map[string]string{ - "/file": "file 0644 5b41362b", + "/dir/": "dir 0755", + "/dir/other-file": "file 0644 63d5dd49", }, manifestPaths: map[string]string{ - "/file": "file 0644 5b41362b {test-package_slice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", }, }, { - summary: "Upgrade keeps untracked files", - slices: []setup.SliceKey{{"test-package", "slice1"}}, + summary: "Upgrade restores modified content and mode", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: slice1: contents: - /file: {text: data1} + /dir/file: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - pkg := release.Packages["test-package"] - slice1 := pkg.Slices["slice1"] - manifestSlice := pkg.Slices["manifest"] - writeFile(c, opts.TargetDir, "/file", []byte("data1"), 0o644) - report, err := manifestutil.NewReport(opts.TargetDir) - c.Assert(err, IsNil) - err = report.Add(slice1, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/file"), - Mode: 0o644, - SHA256: "5b41362b", - Size: 5, - }) - c.Assert(err, IsNil) - err = report.Add(manifestSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, manifestPath), - Mode: 0o644, - }) + alterFilesystem: func(c *C, targetDir string) { + modifiedPath := filepath.Join(targetDir, "dir/file") + err := os.WriteFile(modifiedPath, []byte("data2"), 0o700) c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, pkg.Name, []*setup.Slice{slice1, manifestSlice}, report) - err = os.MkdirAll(filepath.Join(opts.TargetDir, "extra"), 0o755) - c.Assert(err, IsNil) - err = os.WriteFile(filepath.Join(opts.TargetDir, "extra", "untracked"), []byte("data"), 0o644) - c.Assert(err, IsNil) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) }, filesystem: map[string]string{ - "/extra/": "dir 0755", - "/extra/untracked": "file 0644 3a6eb079", - "/file": "file 0644 5b41362b", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", }, manifestPaths: map[string]string{ - "/file": "file 0644 5b41362b {test-package_slice1}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", }, }, { - summary: "Upgrade creates parent directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade keeps untracked files", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - myslice: + slice1: contents: - /foo/bar: {text: data} + /dir/file: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + prefill: func(c *C, targetDir string) { + err := os.MkdirAll(filepath.Join(targetDir, "extra"), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(targetDir, "extra", "untracked"), []byte("data"), 0o644) + c.Assert(err, IsNil) }, filesystem: map[string]string{ - "/foo/": "dir 0755", - "/foo/bar": "file 0644 3a6eb079", + "/extra/": "dir 0755", + "/extra/untracked": "file 0644 3a6eb079", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", }, manifestPaths: map[string]string{ - "/foo/bar": "file 0644 3a6eb079 {test-package_myslice}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", }, }, { - summary: "Upgrade creates symlink", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade does not override existing directory mode", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - myslice: + slice1: contents: - /baz: {text: data} - /foo: {symlink: baz} - `, - }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, - filesystem: map[string]string{ - "/baz": "file 0644 3a6eb079", - "/foo": "symlink baz", - }, - manifestPaths: map[string]string{ - "/baz": "file 0644 3a6eb079 {test-package_myslice}", - "/foo": "symlink baz {test-package_myslice}", - }, -}, { - summary: "Upgrade does not override existing directory mode", - slices: []setup.SliceKey{{"test-package", "myslice"}}, - release: map[string]string{ - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - myslice: + /dir/: + slice2: contents: - /dir/: {make: true, mode: 0775} + /other-dir/: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - err := os.Mkdir(filepath.Join(opts.TargetDir, "dir"), 0o700) + alterFilesystem: func(c *C, targetDir string) { + err := os.MkdirAll(filepath.Join(targetDir, "other-dir"), 0o775) c.Assert(err, IsNil) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) }, filesystem: map[string]string{ - "/dir/": "dir 0700", + "/dir/": "dir 0755", + "/other-dir/": "dir 0775", }, manifestPaths: map[string]string{ - "/dir/": "dir 0775 {test-package_myslice}", + "/dir/": "dir 0755 {test-package_slice1}", + "/other-dir/": "dir 0755 {test-package_slice2}", + // TODO: fix when the right value is recorded in the manifest }, }, { - summary: "Upgrade overwrites existing file mode", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade overwrites existing file mode", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - myslice: + slice1: + contents: + /other-dir/: + slice2: contents: - /foo: {text: data} + /dir/file: `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - writeFile(c, opts.TargetDir, "/foo", []byte("data"), 0o600) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) + prefill: func(c *C, targetDir string) { + err := os.MkdirAll(filepath.Join(targetDir, "dir"), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(targetDir, "dir", "file"), []byte("data"), 0o644) + c.Assert(err, IsNil) }, filesystem: map[string]string{ - "/foo": "file 0644 3a6eb079", + "/dir/": "dir 0755", + "/other-dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", }, manifestPaths: map[string]string{ - "/foo": "file 0644 3a6eb079 {test-package_myslice}", + "/other-dir/": "dir 0755 {test-package_slice1}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice2}", }, }, { - summary: "Upgrade overwrites existing symlink", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade overwrites existing symlink", + cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2234,13 +2367,6 @@ var slicerTests = []slicerTest{{ /foo: {symlink: baz} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - writeFile(c, opts.TargetDir, "/bar", []byte("data"), 0o644) - c.Assert(os.Symlink("bar", filepath.Join(opts.TargetDir, "foo")), IsNil) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, filesystem: map[string]string{ "/bar": "file 0644 3a6eb079", "/baz": "file 0644 3a6eb079", @@ -2251,8 +2377,8 @@ var slicerTests = []slicerTest{{ "/foo": "symlink baz {test-package_myslice}", }, }, { - summary: "Upgrade fails when parent is a file", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade fails when parent is a file", + cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2262,16 +2388,10 @@ var slicerTests = []slicerTest{{ /dir/file: {text: data} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - writeFile(c, opts.TargetDir, "/dir", []byte("data"), 0o644) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, error: `mkdir .*: not a directory`, }, { - summary: "Upgrade fails when target path is a directory", - slices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade fails when target path is a directory", + cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2281,16 +2401,10 @@ var slicerTests = []slicerTest{{ /target: {text: data} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "target"), 0o755), IsNil) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, error: `rename .*: (file exists|is a directory)`, }, { - summary: "Upgrade removes obsolete empty directory", - slices: []setup.SliceKey{{"test-package", "new"}}, + summary: "Upgrade removes obsolete empty directory", + cutSlices: []setup.SliceKey{{"test-package", "new"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2303,18 +2417,6 @@ var slicerTests = []slicerTest{{ /new-file: {text: data1} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "old-dir"), 0o755), IsNil) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - oldSlice := release.Packages["test-package"].Slices["old"] - err := report.Add(oldSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/old-dir"), - Mode: fs.ModeDir | 0o755, - }) - c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, filesystem: map[string]string{ "/new-file": "file 0644 5b41362b", }, @@ -2322,8 +2424,8 @@ var slicerTests = []slicerTest{{ "/new-file": "file 0644 5b41362b {test-package_new}", }, }, { - summary: "Upgrade keeps obsolete non-empty directory", - slices: []setup.SliceKey{{"test-package", "new"}}, + summary: "Upgrade keeps obsolete non-empty directory", + cutSlices: []setup.SliceKey{{"test-package", "new"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2336,19 +2438,6 @@ var slicerTests = []slicerTest{{ /new-file: {text: data1} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - c.Assert(os.MkdirAll(filepath.Join(opts.TargetDir, "old-dir"), 0o755), IsNil) - writeFile(c, opts.TargetDir, "/old-dir/file", []byte("data"), 0o644) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - oldSlice := release.Packages["test-package"].Slices["old"] - err := report.Add(oldSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/old-dir"), - Mode: fs.ModeDir | 0o755, - }) - c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, filesystem: map[string]string{ "/new-file": "file 0644 5b41362b", "/old-dir/": "dir 0755", @@ -2358,8 +2447,8 @@ var slicerTests = []slicerTest{{ "/new-file": "file 0644 5b41362b {test-package_new}", }, }, { - summary: "Upgrade removes obsolete symlink only", - slices: []setup.SliceKey{{"test-package", "new"}}, + summary: "Upgrade removes obsolete symlink only", + cutSlices: []setup.SliceKey{{"test-package", "new"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2372,64 +2461,189 @@ var slicerTests = []slicerTest{{ /new-file: {text: data1} `, }, - prefill: func(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath string) { - writeFile(c, opts.TargetDir, "/target", []byte("data"), 0o644) - c.Assert(os.Symlink("target", filepath.Join(opts.TargetDir, "link")), IsNil) - report, manifestSlice := newManifestReport(c, opts, release, manifestPath, "test-package") - oldSlice := release.Packages["test-package"].Slices["old"] - err := report.Add(oldSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, "/link"), - Mode: fs.ModeSymlink, - Link: "target", + filesystem: map[string]string{ + "/new-file": "file 0644 5b41362b", + "/target": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/new-file": "file 0644 5b41362b {test-package_new}", + }, +}} + +func (s *S) TestRunRecut(c *C) { + for _, test := range slicerRecutTests { + const logMarker = "---log-marker---" + c.Logf(logMarker) + c.Logf("Summary: %s", test.summary) + + if _, ok := test.release["chisel.yaml"]; !ok { + test.release["chisel.yaml"] = testutil.DefaultChiselYaml + } + if test.pkgs == nil { + test.pkgs = []*testutil.TestPackage{{ + Name: "test-package", + Data: testutil.PackageData["test-package"], + }} + } + for _, pkg := range test.pkgs { + // We need to set these fields for manifest validation. + if pkg.Arch == "" { + pkg.Arch = "arch" + } + if pkg.Hash == "" { + pkg.Hash = "hash" + } + if pkg.Version == "" { + pkg.Version = "version" + } + } + + releaseDir := c.MkDir() + for path, data := range test.release { + fpath := filepath.Join(releaseDir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0o755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0o644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(releaseDir) + c.Assert(err, IsNil) + + // Create a manifest slice and add it to the selection. + manifestPackage := test.cutSlices[0].Package + manifestPath := "/chisel-data/manifest.wall" + release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ + Package: manifestPackage, + Name: "manifest", + Essential: nil, + Contents: map[string]setup.PathInfo{ + "/chisel-data/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + Scripts: setup.SliceScripts{}, + } + test.cutSlices = append(test.cutSlices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", }) - c.Assert(err, IsNil) - writeManifestReport(c, opts.TargetDir, manifestPath, "test-package", []*setup.Slice{oldSlice, manifestSlice}, report) - opts.Manifest = readManifest(c, opts.TargetDir, manifestPath) - }, - filesystem: map[string]string{ - "/new-file": "file 0644 5b41362b", - "/target": "file 0644 3a6eb079", - }, - manifestPaths: map[string]string{ - "/new-file": "file 0644 5b41362b {test-package_new}", - }, -}} -func (s *S) TestRun(c *C) { - // Run tests for "archives" field in "v1" format. - runSlicerTests(s, c, slicerTests) + selection, err := setup.Select(release, test.cutSlices, test.arch) + c.Assert(err, IsNil) - // Run tests for "v2-archives" field in "v1" format. - v2ArchiveTests := make([]slicerTest, 0, len(slicerTests)) - for _, t := range slicerTests { - m := make(map[string]string) - for k, v := range t.release { - if !strings.Contains(v, "v2-archives:") { - v = strings.Replace(v, "archives:", "v2-archives:", -1) + archives := map[string]archive.Archive{} + for name, setupArchive := range release.Archives { + pkgs := make(map[string]*testutil.TestPackage) + for _, pkg := range test.pkgs { + if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { + pkgs[pkg.Name] = pkg + } } - m[k] = v + archive := &testutil.TestArchive{ + Opts: archive.Options{ + Label: setupArchive.Name, + Version: setupArchive.Version, + Suites: setupArchive.Suites, + Components: setupArchive.Components, + Pro: setupArchive.Pro, + Arch: test.arch, + }, + Packages: pkgs, + } + archives[name] = archive } - t.release = m - v2ArchiveTests = append(v2ArchiveTests, t) - } - runSlicerTests(s, c, v2ArchiveTests) - // Run tests for "v2" format. - v2FormatTests := make([]slicerTest, 0, len(slicerTests)) - for _, t := range slicerTests { - m := make(map[string]string) - for k, v := range t.release { - if strings.Contains(v, "format: v1") && - !strings.Contains(v, "v2-archives:") && - !strings.Contains(v, "default: true") { - v = strings.Replace(v, "format: v1", "format: v2", -1) - } - m[k] = v + targetDir := c.MkDir() + options := slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: targetDir, + } + if test.hackopt != nil { + test.hackopt(c, &options) + } + if test.prefill != nil { + test.prefill(c, targetDir) + } + // First run + err = slicer.Run(&options) + c.Assert(err, IsNil) + + if test.alterFilesystem != nil { + test.alterFilesystem(c, targetDir) + } + mfest := readManifest(c, options.TargetDir, manifestPath) + + test.recutSlices = append(test.recutSlices, setup.SliceKey{ + Package: manifestPackage, + Slice: "manifest", + }) + selection, err = setup.Select(release, test.recutSlices, test.arch) + c.Assert(err, IsNil) + + options = slicer.RunOptions{ + Selection: selection, + Archives: archives, + TargetDir: targetDir, + Manifest: mfest, + } + if test.hackRecutOpt != nil { + test.hackRecutOpt(c, &options) + } + // Second run + err = slicer.Run(&options) + if test.error != "" { + c.Assert(err, ErrorMatches, test.error) + continue + } + c.Assert(err, IsNil) + + if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil && test.logOutput == "" { + continue + } + mfest = readManifest(c, options.TargetDir, manifestPath) + + // Assert state of final filesystem. + if test.filesystem != nil { + filesystem := testutil.TreeDump(options.TargetDir) + c.Assert(filesystem["/chisel-data/"], Not(HasLen), 0) + c.Assert(filesystem[manifestPath], Not(HasLen), 0) + delete(filesystem, "/chisel-data/") + delete(filesystem, manifestPath) + c.Assert(filesystem, DeepEquals, test.filesystem) + } + + // Assert state of the files recorded in the manifest. + if test.manifestPaths != nil { + pathsDump, err := treeDumpManifestPaths(mfest) + c.Assert(err, IsNil) + c.Assert(pathsDump[manifestPath], Not(HasLen), 0) + delete(pathsDump, manifestPath) + c.Assert(pathsDump, DeepEquals, test.manifestPaths) + } + + // Assert state of the packages recorded in the manifest. + if test.manifestPkgs != nil { + pkgsDump, err := dumpManifestPkgs(mfest) + c.Assert(err, IsNil) + c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) + } + + // Find the log output of this test by trimming the suite output + // until we find the last occurrence of the summary. + testLogs := strings.Split(c.GetTestLog(), logMarker) + logOutput := testLogs[len(testLogs)-1] + + // Assert log output. + if test.logOutput != "" { + c.Assert(logOutput, Matches, test.logOutput) + } else { + // No warnings emitted. + c.Assert(logOutput, Not(Matches), "(?s).*Warning.*") } - t.release = m - v2FormatTests = append(v2FormatTests, t) } - runSlicerTests(s, c, v2FormatTests) } type selectValidManifestTest struct { @@ -2459,7 +2673,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ }, setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := manifestPathForDir("/chisel/**") - writeManifest(c, targetDir, manifestPath, releaseManifestSlice(release), "hash1") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPath, slice, "hash1") }, }, { summary: "Two consistent manifests are accepted", @@ -2469,7 +2684,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ setup: func(c *C, targetDir string, release *setup.Release) { manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") - slice := releaseManifestSlice(release) + slice := release.Packages["test-package"].Slices["manifest"] writeManifest(c, targetDir, manifestPathA, slice, "hash1") writeManifest(c, targetDir, manifestPathB, slice, "hash1") }, @@ -2481,7 +2696,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ setup: func(c *C, targetDir string, release *setup.Release) { manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") - slice := releaseManifestSlice(release) + slice := release.Packages["test-package"].Slices["manifest"] writeManifest(c, targetDir, manifestPathA, slice, "hash1") writeManifest(c, targetDir, manifestPathB, slice, "hash2") }, @@ -2540,158 +2755,6 @@ func (s *S) TestSelectValidManifest(c *C) { continue } c.Assert(mfest, NotNil) - c.Assert(mfest.Schema(), Equals, manifest.Schema) - } -} - -func runSlicerTests(s *S, c *C, tests []slicerTest) { - for _, test := range tests { - for _, testSlices := range testutil.Permutations(test.slices) { - const logMarker = "---log-marker---" - c.Logf(logMarker) - c.Logf("Summary: %s", test.summary) - - if _, ok := test.release["chisel.yaml"]; !ok { - test.release["chisel.yaml"] = testutil.DefaultChiselYaml - } - if test.pkgs == nil { - test.pkgs = []*testutil.TestPackage{{ - Name: "test-package", - Data: testutil.PackageData["test-package"], - }} - } - for _, pkg := range test.pkgs { - // We need to set these fields for manifest validation. - if pkg.Arch == "" { - pkg.Arch = "arch" - } - if pkg.Hash == "" { - pkg.Hash = "hash" - } - if pkg.Version == "" { - pkg.Version = "version" - } - } - - releaseDir := c.MkDir() - for path, data := range test.release { - fpath := filepath.Join(releaseDir, path) - err := os.MkdirAll(filepath.Dir(fpath), 0755) - c.Assert(err, IsNil) - err = os.WriteFile(fpath, testutil.Reindent(data), 0644) - c.Assert(err, IsNil) - } - - release, err := setup.ReadRelease(releaseDir) - c.Assert(err, IsNil) - - // Create a manifest slice and add it to the selection. - manifestPackage := test.slices[0].Package - manifestPath := "/chisel-data/manifest.wall" - release.Packages[manifestPackage].Slices["manifest"] = &setup.Slice{ - Package: manifestPackage, - Name: "manifest", - Essential: nil, - Contents: map[string]setup.PathInfo{ - "/chisel-data/**": { - Kind: "generate", - Generate: "manifest", - }, - }, - Scripts: setup.SliceScripts{}, - } - testSlices = append(testSlices, setup.SliceKey{ - Package: manifestPackage, - Slice: "manifest", - }) - - selection, err := setup.Select(release, testSlices, test.arch) - c.Assert(err, IsNil) - - archives := map[string]archive.Archive{} - for name, setupArchive := range release.Archives { - pkgs := make(map[string]*testutil.TestPackage) - for _, pkg := range test.pkgs { - if len(pkg.Archives) == 0 || slices.Contains(pkg.Archives, name) { - pkgs[pkg.Name] = pkg - } - } - archive := &testutil.TestArchive{ - Opts: archive.Options{ - Label: setupArchive.Name, - Version: setupArchive.Version, - Suites: setupArchive.Suites, - Components: setupArchive.Components, - Pro: setupArchive.Pro, - Arch: test.arch, - }, - Packages: pkgs, - } - archives[name] = archive - } - - options := slicer.RunOptions{ - Selection: selection, - Archives: archives, - TargetDir: c.MkDir(), - } - if test.hackopt != nil { - test.hackopt(c, &options) - } - if test.prefill != nil { - test.prefill(c, &options, release, manifestPath) - } - err = slicer.Run(&options) - if test.error != "" { - c.Assert(err, ErrorMatches, test.error) - continue - } - c.Assert(err, IsNil) - - if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil && test.logOutput == "" { - continue - } - mfest := readManifest(c, options.TargetDir, manifestPath) - - // Assert state of final filesystem. - if test.filesystem != nil { - filesystem := testutil.TreeDump(options.TargetDir) - c.Assert(filesystem["/chisel-data/"], Not(HasLen), 0) - c.Assert(filesystem[manifestPath], Not(HasLen), 0) - delete(filesystem, "/chisel-data/") - delete(filesystem, manifestPath) - c.Assert(filesystem, DeepEquals, test.filesystem) - } - - // Assert state of the files recorded in the manifest. - if test.manifestPaths != nil { - pathsDump, err := treeDumpManifestPaths(mfest) - c.Assert(err, IsNil) - c.Assert(pathsDump[manifestPath], Not(HasLen), 0) - delete(pathsDump, manifestPath) - c.Assert(pathsDump, DeepEquals, test.manifestPaths) - } - - // Assert state of the packages recorded in the manifest. - if test.manifestPkgs != nil { - pkgsDump, err := dumpManifestPkgs(mfest) - c.Assert(err, IsNil) - c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) - } - - // Find the log output of this test by trimming the suite output - // until we find the last occurrence of the summary. - testLogs := strings.Split(c.GetTestLog(), logMarker) - logOutput := testLogs[len(testLogs)-1] - - // Assert log output. - if test.logOutput != "" { - c.Assert(logOutput, Matches, test.logOutput) - } else { - // No warnings emitted. - c.Assert(logOutput, Not(Matches), "(?s).*Warning.*") - } - } } } @@ -2716,10 +2779,6 @@ func buildReleaseWithManifestDirs(dirs ...string) *setup.Release { } } -func releaseManifestSlice(release *setup.Release) *setup.Slice { - return release.Packages["test-package"].Slices["manifest"] -} - func manifestPathForDir(dir string) string { base := strings.TrimSuffix(dir, "**") return path.Join(base, manifestutil.DefaultFilename) @@ -2802,18 +2861,6 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { return mfest } -func newManifestReport(c *C, opts *slicer.RunOptions, release *setup.Release, manifestPath, pkgName string) (*manifestutil.Report, *setup.Slice) { - report, err := manifestutil.NewReport(opts.TargetDir) - c.Assert(err, IsNil) - manifestSlice := release.Packages[pkgName].Slices["manifest"] - err = report.Add(manifestSlice, &fsutil.Entry{ - Path: filepath.Join(report.Root, manifestPath), - Mode: 0o644, - }) - c.Assert(err, IsNil) - return report, manifestSlice -} - func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { mfestPath := filepath.Join(targetDir, manifestPath) err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) @@ -2847,31 +2894,6 @@ func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, has c.Assert(os.Chmod(mfestPath, 0o644), IsNil) } -func writeManifestReport(c *C, targetDir, manifestPath, pkgName string, selection []*setup.Slice, report *manifestutil.Report) { - mfestPath := filepath.Join(targetDir, manifestPath) - err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) - c.Assert(err, IsNil) - f, err := os.OpenFile(mfestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644) - c.Assert(err, IsNil) - zw, err := zstd.NewWriter(f) - c.Assert(err, IsNil) - options := &manifestutil.WriteOptions{ - PackageInfo: []*archive.PackageInfo{{ - Name: pkgName, - Version: "1.0", - Arch: "amd64", - SHA256: "pkg-hash", - }}, - Selection: selection, - Report: report, - } - err = manifestutil.Write(options, zw) - c.Assert(err, IsNil) - c.Assert(zw.Close(), IsNil) - c.Assert(f.Close(), IsNil) - c.Assert(os.Chmod(mfestPath, 0o644), IsNil) -} - func writeInvalidManifest(c *C, targetDir, manifestPath string) { mfestPath := filepath.Join(targetDir, manifestPath) err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) @@ -2903,11 +2925,3 @@ func writeInvalidSchemaManifest(c *C, targetDir, manifestPath string) { c.Assert(zw.Close(), IsNil) c.Assert(f.Close(), IsNil) } - -func writeFile(c *C, targetDir, relPath string, data []byte, mode fs.FileMode) { - path := filepath.Join(targetDir, relPath) - err := os.MkdirAll(filepath.Dir(path), 0o755) - c.Assert(err, IsNil) - err = os.WriteFile(path, data, mode) - c.Assert(err, IsNil) -} From f8e2bd64af3fb2e77d7e9e68a4c81a475ead71fc Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 11 Feb 2026 13:49:17 +0100 Subject: [PATCH 38/65] fix: ignore unknown manifest schema error --- internal/slicer/slicer.go | 3 +++ internal/slicer/slicer_test.go | 10 ++++++++++ public/manifest/manifest.go | 6 ++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 87998b31..350f2492 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -664,6 +664,9 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma defer r.Close() mfest, err := manifest.Read(r) if err != nil { + if errors.Is(err, manifest.ErrUnknownSchema) { + continue + } return nil, err } err = manifestutil.Validate(mfest) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index a7c35f7e..78f33122 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2666,6 +2666,16 @@ var selectValidManifestTests = []selectValidManifestTest{{ return buildReleaseWithManifestDirs("/chisel/**") }, noMatch: true, +}, { + summary: "Unknown schema error ignored", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPath := manifestPathForDir("/chisel/**") + writeInvalidSchemaManifest(c, targetDir, manifestPath) + }, + noMatch: true, }, { summary: "Valid manifest selected", build: func() *setup.Release { diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..5a1bb34c 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -46,12 +46,14 @@ type Manifest struct { db *jsonwall.DB } +var ErrUnknownSchema = fmt.Errorf("unknown schema version") + // Read loads a Manifest without performing any validation. The data is assumed // to be both valid jsonwall and a valid Manifest (see Validate). func Read(reader io.Reader) (manifest *Manifest, err error) { defer func() { if err != nil { - err = fmt.Errorf("cannot read manifest: %s", err) + err = fmt.Errorf("cannot read manifest: %w", err) } }() @@ -61,7 +63,7 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { } mfestSchema := db.Schema() if mfestSchema != Schema { - return nil, fmt.Errorf("unknown schema version %q", mfestSchema) + return nil, fmt.Errorf("%w %q", ErrUnknownSchema, mfestSchema) } manifest = &Manifest{db: db} From 6dd701cfcf501951fc01aac83a038522448208ff Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 11 Feb 2026 15:35:00 +0100 Subject: [PATCH 39/65] feat: improve upgrade and add tests --- internal/slicer/slicer.go | 80 ++++++++++++---- internal/slicer/slicer_test.go | 166 ++++++++++++++------------------- 2 files changed, 130 insertions(+), 116 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 350f2492..363c4d85 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -112,7 +112,7 @@ func Run(options *RunOptions) error { installOpts := &optsCopy installOpts.TargetDir = targetDir - report, pkgInfos, err := install(installOpts) + report, err := install(installOpts) if err != nil { return err } @@ -124,18 +124,18 @@ func Run(options *RunOptions) error { } } - return generateManifests(originalTargetDir, options.Selection, report, pkgInfos) + return nil } -func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, error) { +func install(options *RunOptions) (*manifestutil.Report, error) { pkgArchive, err := selectPkgArchives(options.Archives, options.Selection) if err != nil { - return nil, nil, err + return nil, err } prefers, err := options.Selection.Prefers() if err != nil { - return nil, nil, err + return nil, err } // Build information to process the selection. @@ -192,7 +192,7 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, } reader, info, err := pkgArchive[slice.Package].Fetch(slice.Package) if err != nil { - return nil, nil, err + return nil, err } defer reader.Close() packages[slice.Package] = reader @@ -205,7 +205,7 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, addKnownPath(knownPaths, "/", pathData{}) report, err := manifestutil.NewReport(options.TargetDir) if err != nil { - return nil, nil, fmt.Errorf("internal error: cannot create report: %w", err) + return nil, fmt.Errorf("internal error: cannot create report: %w", err) } // Record directories which are created but where not listed in the slice @@ -285,7 +285,7 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, reader.Close() packages[slice.Package] = nil if err != nil { - return nil, nil, err + return nil, err } } @@ -343,7 +343,7 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, addKnownPath(knownPaths, relPath, data) entry, err := createFile(options.TargetDir, relPath, pathInfo) if err != nil { - return nil, nil, err + return nil, err } // Do not add paths with "until: mutate". @@ -351,7 +351,7 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, for _, slice := range slices { err = report.Add(slice, entry) if err != nil { - return nil, nil, err + return nil, err } } } @@ -376,16 +376,21 @@ func install(options *RunOptions) (*manifestutil.Report, []*archive.PackageInfo, } err := scripts.Run(&opts) if err != nil { - return nil, nil, fmt.Errorf("slice %s: %w", slice, err) + return nil, fmt.Errorf("slice %s: %w", slice, err) } } err = removeAfterMutate(options.TargetDir, knownPaths) if err != nil { - return nil, nil, err + return nil, err } - return report, pkgInfos, nil + err = generateManifests(options.TargetDir, options.Selection, report, pkgInfos) + if err != nil { + return nil, err + } + + return report, nil } // upgrade upgrades content in targetDir with content in tempDir. @@ -402,12 +407,9 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes var err error switch entry.Mode & fs.ModeType { case 0, fs.ModeSymlink: - err = os.Rename(srcPath, dstPath) + err = upgradeFile(srcPath, dstPath) case fs.ModeDir: - mkdirErr := os.Mkdir(dstPath, entry.Mode) - if !os.IsExist(mkdirErr) { - err = mkdirErr - } + err = upgradeDir(dstPath, entry) default: err = fmt.Errorf("unsupported file type: %s", path) } @@ -429,6 +431,9 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes return err } sort.Sort(sort.Reverse(sort.StringSlice(missingPaths))) + // Go through the list in reverse order to empty directories before removing + // them. Any ENOTEMPTY error encountered means user content is in the directory + // and Chisel does not manage it anymore. for _, relPath := range missingPaths { path := filepath.Clean(filepath.Join(targetDir, relPath)) err = os.Remove(path) @@ -436,7 +441,44 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes return err } } - report.Root = targetDir + return nil +} + +func upgradeDir(path string, entry manifestutil.ReportEntry) error { + err := os.Mkdir(path, entry.Mode) + if err != nil { + if !os.IsExist(err) { + return err + } + fileinfo, err := os.Lstat(path) + if err != nil { + return err + } + if fileinfo.IsDir() { + return os.Chmod(path, entry.Mode) + } + err = os.RemoveAll(path) + if err != nil { + return err + } + + return os.Mkdir(path, entry.Mode) + } + return nil +} + +func upgradeFile(srcPath string, dstPath string) error { + err := os.Rename(srcPath, dstPath) + if err != nil { + if !os.IsExist(err) { + return err + } + err = os.RemoveAll(dstPath) + if err != nil { + return err + } + return os.Rename(srcPath, dstPath) + } return nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 78f33122..d12562d5 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2295,7 +2295,7 @@ var slicerRecutTests = []slicerRecutTest{{ "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", }, }, { - summary: "Upgrade does not override existing directory mode", + summary: "Upgrade overrides existing mode", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ @@ -2307,24 +2307,28 @@ var slicerRecutTests = []slicerRecutTest{{ /dir/: slice2: contents: + /dir/file: /other-dir/: `, }, alterFilesystem: func(c *C, targetDir string) { err := os.MkdirAll(filepath.Join(targetDir, "other-dir"), 0o775) c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(targetDir, "dir", "file"), []byte("data"), 0o644) + c.Assert(err, IsNil) }, filesystem: map[string]string{ "/dir/": "dir 0755", - "/other-dir/": "dir 0775", + "/other-dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", }, manifestPaths: map[string]string{ "/dir/": "dir 0755 {test-package_slice1}", "/other-dir/": "dir 0755 {test-package_slice2}", - // TODO: fix when the right value is recorded in the manifest + "/dir/file": "file 0644 cc55e2ec {test-package_slice2}", }, }, { - summary: "Upgrade overwrites existing file mode", + summary: "Upgrade overwrites existing symlink", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ @@ -2333,140 +2337,118 @@ var slicerRecutTests = []slicerRecutTest{{ slices: slice1: contents: - /other-dir/: + /dir/: slice2: contents: - /dir/file: + /baz: {text: data} + /foo: {symlink: baz} `, }, - prefill: func(c *C, targetDir string) { - err := os.MkdirAll(filepath.Join(targetDir, "dir"), 0o755) + alterFilesystem: func(c *C, targetDir string) { + err := os.WriteFile(filepath.Join(targetDir, "bar"), []byte("data"), 0o644) c.Assert(err, IsNil) - err = os.WriteFile(filepath.Join(targetDir, "dir", "file"), []byte("data"), 0o644) + linkPath := filepath.Join(targetDir, "foo") + err = os.Symlink("bar", linkPath) c.Assert(err, IsNil) }, filesystem: map[string]string{ - "/dir/": "dir 0755", - "/other-dir/": "dir 0755", - "/dir/file": "file 0644 cc55e2ec", + "/dir/": "dir 0755", + "/bar": "file 0644 3a6eb079", + "/baz": "file 0644 3a6eb079", + "/foo": "symlink baz", }, manifestPaths: map[string]string{ - "/other-dir/": "dir 0755 {test-package_slice1}", - "/dir/file": "file 0644 cc55e2ec {test-package_slice2}", + "/dir/": "dir 0755 {test-package_slice1}", + "/baz": "file 0644 3a6eb079 {test-package_slice2}", + "/foo": "symlink baz {test-package_slice2}", }, }, { - summary: "Upgrade overwrites existing symlink", - cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, - release: map[string]string{ - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - myslice: - contents: - /baz: {text: data} - /foo: {symlink: baz} - `, - }, - filesystem: map[string]string{ - "/bar": "file 0644 3a6eb079", - "/baz": "file 0644 3a6eb079", - "/foo": "symlink baz", - }, - manifestPaths: map[string]string{ - "/baz": "file 0644 3a6eb079 {test-package_myslice}", - "/foo": "symlink baz {test-package_myslice}", - }, -}, { - summary: "Upgrade fails when parent is a file", - cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade fails when parent is a file", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - myslice: + slice1: contents: /dir/file: {text: data} + slice2: + contents: `, }, + alterFilesystem: func(c *C, targetDir string) { + path := filepath.Join(targetDir, "dir") + err := os.RemoveAll(path) + c.Assert(err, IsNil) + err = os.WriteFile(path, []byte("data"), 0o644) + c.Assert(err, IsNil) + }, error: `mkdir .*: not a directory`, }, { - summary: "Upgrade fails when target path is a directory", - cutSlices: []setup.SliceKey{{"test-package", "myslice"}}, + summary: "Upgrade removes content whith unmatching type", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - myslice: + slice1: contents: - /target: {text: data} + /file: {text: data} + /a-dir/: {make: true} `, }, - error: `rename .*: (file exists|is a directory)`, -}, { - summary: "Upgrade removes obsolete empty directory", - cutSlices: []setup.SliceKey{{"test-package", "new"}}, - release: map[string]string{ - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - old: - contents: - /old-dir/: {make: true} - new: - contents: - /new-file: {text: data1} - `, + alterFilesystem: func(c *C, targetDir string) { + filePath := filepath.Join(targetDir, "file") + err := os.Remove(filePath) + c.Assert(err, IsNil) + err = os.MkdirAll(filePath, 0o755) + c.Assert(err, IsNil) + dirPath := filepath.Join(targetDir, "a-dir") + err = os.Remove(dirPath) + c.Assert(err, IsNil) + err = os.WriteFile(dirPath, []byte("data"), 0o644) + c.Assert(err, IsNil) }, filesystem: map[string]string{ - "/new-file": "file 0644 5b41362b", + "/file": "file 0644 3a6eb079", + "/a-dir/": "dir 0755", }, manifestPaths: map[string]string{ - "/new-file": "file 0644 5b41362b {test-package_new}", + "/file": "file 0644 3a6eb079 {test-package_slice1}", + "/a-dir/": "dir 0755 {test-package_slice1}", }, }, { - summary: "Upgrade keeps obsolete non-empty directory", - cutSlices: []setup.SliceKey{{"test-package", "new"}}, + summary: "Upgrade removes obsolete content but keeps non-empty directories", + cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice2"}}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package slices: - old: + slice1: contents: /old-dir/: {make: true} - new: + /link: {symlink: target} + /baz: {text: data} + /foo: {symlink: baz} + slice2: contents: /new-file: {text: data1} `, }, + alterFilesystem: func(c *C, targetDir string) { + err := os.WriteFile(filepath.Join(targetDir, "old-dir", "file"), []byte("data"), 0o644) + c.Assert(err, IsNil) + }, filesystem: map[string]string{ "/new-file": "file 0644 5b41362b", "/old-dir/": "dir 0755", "/old-dir/file": "file 0644 3a6eb079", }, manifestPaths: map[string]string{ - "/new-file": "file 0644 5b41362b {test-package_new}", - }, -}, { - summary: "Upgrade removes obsolete symlink only", - cutSlices: []setup.SliceKey{{"test-package", "new"}}, - release: map[string]string{ - "slices/mydir/test-package.yaml": ` - package: test-package - slices: - old: - contents: - /link: {symlink: target} - new: - contents: - /new-file: {text: data1} - `, - }, - filesystem: map[string]string{ - "/new-file": "file 0644 5b41362b", - "/target": "file 0644 3a6eb079", - }, - manifestPaths: map[string]string{ - "/new-file": "file 0644 5b41362b {test-package_new}", + "/new-file": "file 0644 5b41362b {test-package_slice2}", }, }} @@ -2734,16 +2716,6 @@ var selectValidManifestTests = []selectValidManifestTest{{ writeInvalidManifest(c, targetDir, manifestPath) }, error: `invalid manifest: path /file has no matching entry in contents`, -}, { - summary: "Manifest read fails on invalid schema", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, - setup: func(c *C, targetDir string, release *setup.Release) { - manifestPath := manifestPathForDir("/chisel/**") - writeInvalidSchemaManifest(c, targetDir, manifestPath) - }, - error: `cannot read manifest: unknown schema version "9.9"`, }} func (s *S) TestSelectValidManifest(c *C) { From 8523aa5472d4544db0c77227ca40187f1ff9fdc5 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 11 Feb 2026 15:38:39 +0100 Subject: [PATCH 40/65] tests: simplify TestRunRecut --- internal/slicer/slicer_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d12562d5..7a5a52a3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2175,7 +2175,6 @@ type slicerRecutTest struct { recutSlices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) hackRecutOpt func(c *C, opts *slicer.RunOptions) - prefill func(c *C, targetDir string) alterFilesystem func(c *C, targetDir string) // Modifies the filesystem built after the first execution and before the // second one. @@ -2279,7 +2278,7 @@ var slicerRecutTests = []slicerRecutTest{{ /dir/file: `, }, - prefill: func(c *C, targetDir string) { + alterFilesystem: func(c *C, targetDir string) { err := os.MkdirAll(filepath.Join(targetDir, "extra"), 0o755) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(targetDir, "extra", "untracked"), []byte("data"), 0o644) @@ -2546,9 +2545,6 @@ func (s *S) TestRunRecut(c *C) { if test.hackopt != nil { test.hackopt(c, &options) } - if test.prefill != nil { - test.prefill(c, targetDir) - } // First run err = slicer.Run(&options) c.Assert(err, IsNil) From 2ceb21acd169359ab3a56128c1f4751b7f71f232 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 12 Feb 2026 13:05:53 +0100 Subject: [PATCH 41/65] fix: rename recut env var --- cmd/chisel/cmd_cut.go | 2 +- tests/recut/task.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 1d0abfd9..4e9356c0 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -77,7 +77,7 @@ func (cmd *cmdCut) Execute(args []string) error { var mfest *manifest.Manifest // TODO: Remove this gating once the final upgrading strategy is in place. - if os.Getenv("CHISEL_RECUT") != "" { + if os.Getenv("CHISEL_RECUT_EXPERIMENTAL") != "" { mfest, err := slicer.SelectValidManifest(cmd.RootDir, release) if err != nil { return err diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index bdffcc6e..b87119c3 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -14,7 +14,7 @@ execute: | mkdir -p ${ROOTFS} # TODO: remove this env var when final upgrade strategy is in place. - export CHISEL_RECUT=1 + export CHISEL_RECUT_EXPERIMENTAL=1 # First cut generates manifest and installs slice-a. chisel cut --release ./chisel-releases/ \ --root ${ROOTFS} \ From 10a1744afcc6f410e5eb92ff9fbeeb0e638690b9 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 12 Feb 2026 13:22:29 +0100 Subject: [PATCH 42/65] docs: add TODO on other file creation bug --- internal/fsutil/create.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/fsutil/create.go b/internal/fsutil/create.go index 8c5b58e6..08e58828 100644 --- a/internal/fsutil/create.go +++ b/internal/fsutil/create.go @@ -171,7 +171,7 @@ func createDir(o *CreateOptions) error { // TODO: Detect if existing content is a file. ErrExist is also returned // if a file exists at this path, so returning nil here creates an // inconsistency between our view of the content and the real content on - // disk. + // disk which is a bug that must be fixed. return nil } return err @@ -183,6 +183,9 @@ func createFile(o *CreateOptions) error { if err != nil { return err } + // TODO: Detect if existing content is a symlink and remove it if so. The + // current implementation resolves the symlink and will override the target + // and not the symlink itself which is a bug. file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, o.Mode) if err != nil { return err From 6cf317cc79c6b4c081a574e2fcfe6dc22c890f55 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 12 Feb 2026 13:27:04 +0100 Subject: [PATCH 43/65] refactor: simplify targetDir handling in Run --- internal/slicer/slicer.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 363c4d85..9838961f 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -97,20 +97,19 @@ func Run(options *RunOptions) error { targetDir = filepath.Join(dir, targetDir) } - originalTargetDir := targetDir + optsCopy := *options + installOpts := &optsCopy + installOpts.TargetDir = targetDir if options.Manifest != nil { tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") if err != nil { return fmt.Errorf("cannot create temporary working directory: %w", err) } - targetDir = tmpWorkDir + installOpts.TargetDir = tmpWorkDir defer func() { os.RemoveAll(tmpWorkDir) }() } - optsCopy := *options - installOpts := &optsCopy - installOpts.TargetDir = targetDir report, err := install(installOpts) if err != nil { @@ -118,7 +117,7 @@ func Run(options *RunOptions) error { } if options.Manifest != nil { - err = upgrade(originalTargetDir, targetDir, report, options.Manifest) + err = upgrade(targetDir, installOpts.TargetDir, report, options.Manifest) if err != nil { return err } From 0d1103b9c3ef0f978da74aa7e6d25ee46b7b26d5 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 12 Feb 2026 13:33:09 +0100 Subject: [PATCH 44/65] fix: improve workdir name --- internal/slicer/slicer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9838961f..c4024b27 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -101,7 +101,7 @@ func Run(options *RunOptions) error { installOpts := &optsCopy installOpts.TargetDir = targetDir if options.Manifest != nil { - tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-*") + tmpWorkDir, err := os.MkdirTemp(targetDir, "chisel-workdir-*") if err != nil { return fmt.Errorf("cannot create temporary working directory: %w", err) } From 78e3f5118d09d7f68c83a3b64cc88ebcea2acc80 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 12 Feb 2026 13:39:32 +0100 Subject: [PATCH 45/65] fix: clarify intent to remove existing content --- internal/slicer/slicer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index c4024b27..15a667c4 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -456,7 +456,8 @@ func upgradeDir(path string, entry manifestutil.ReportEntry) error { if fileinfo.IsDir() { return os.Chmod(path, entry.Mode) } - err = os.RemoveAll(path) + // Path is a regular file or symlink, remove it. + err = os.Remove(path) if err != nil { return err } From d14c056248cc6d5f03c32acdb030f82fafc29689 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 08:19:16 +0100 Subject: [PATCH 46/65] fix: apply various PR suggestions --- internal/slicer/slicer.go | 5 +++-- internal/slicer/slicer_test.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 15a667c4..d27552d3 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -707,6 +707,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma mfest, err := manifest.Read(r) if err != nil { if errors.Is(err, manifest.ErrUnknownSchema) { + // Ignore manifests with unknown (potentially future) schema versions. continue } return nil, err @@ -717,7 +718,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma } h, err := contentHash(mfestFullPath) if err != nil { - return nil, fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + return nil, fmt.Errorf("cannot compute hash for %q: %s", mfestFullPath, err) } mfestHash := hex.EncodeToString(h) if selected == nil { @@ -725,7 +726,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma selectedHash = mfestHash selectedPath = mfestPath } else if selectedHash != mfestHash { - return nil, fmt.Errorf("inconsistent manifests: %q and %q", selectedPath, mfestPath) + return nil, fmt.Errorf("cannot select a manifest: %q and %q are inconsistent", selectedPath, mfestPath) } } return selected, nil diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 7a5a52a3..dcf19ca4 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2175,9 +2175,9 @@ type slicerRecutTest struct { recutSlices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) hackRecutOpt func(c *C, opts *slicer.RunOptions) - alterFilesystem func(c *C, targetDir string) // Modifies the filesystem built after the first execution and before the // second one. + alterFilesystem func(c *C, targetDir string) filesystem map[string]string manifestPaths map[string]string manifestPkgs map[string]string @@ -2688,7 +2688,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ writeManifest(c, targetDir, manifestPathA, slice, "hash1") writeManifest(c, targetDir, manifestPathB, slice, "hash2") }, - error: `inconsistent manifests: ".*" and ".*"`, + error: `cannot select a manifest: ".*" and ".*" are inconsistent`, }, { summary: "Invalid manifest data returns error", build: func() *setup.Release { From 4217131c824e7bdcbc1ea7c3655b2138a27bdbde Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 08:30:21 +0100 Subject: [PATCH 47/65] fix: apply PR suggestions --- internal/slicer/slicer_test.go | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index dcf19ca4..a4df83e9 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2174,7 +2174,6 @@ type slicerRecutTest struct { cutSlices []setup.SliceKey recutSlices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) - hackRecutOpt func(c *C, opts *slicer.RunOptions) // Modifies the filesystem built after the first execution and before the // second one. alterFilesystem func(c *C, targetDir string) @@ -2453,8 +2452,6 @@ var slicerRecutTests = []slicerRecutTest{{ func (s *S) TestRunRecut(c *C) { for _, test := range slicerRecutTests { - const logMarker = "---log-marker---" - c.Logf(logMarker) c.Logf("Summary: %s", test.summary) if _, ok := test.release["chisel.yaml"]; !ok { @@ -2545,7 +2542,7 @@ func (s *S) TestRunRecut(c *C) { if test.hackopt != nil { test.hackopt(c, &options) } - // First run + // First run. err = slicer.Run(&options) c.Assert(err, IsNil) @@ -2567,10 +2564,7 @@ func (s *S) TestRunRecut(c *C) { TargetDir: targetDir, Manifest: mfest, } - if test.hackRecutOpt != nil { - test.hackRecutOpt(c, &options) - } - // Second run + // Second run. err = slicer.Run(&options) if test.error != "" { c.Assert(err, ErrorMatches, test.error) @@ -2578,7 +2572,7 @@ func (s *S) TestRunRecut(c *C) { } c.Assert(err, IsNil) - if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil && test.logOutput == "" { + if test.filesystem == nil && test.manifestPaths == nil && test.manifestPkgs == nil { continue } mfest = readManifest(c, options.TargetDir, manifestPath) @@ -2608,19 +2602,6 @@ func (s *S) TestRunRecut(c *C) { c.Assert(err, IsNil) c.Assert(pkgsDump, DeepEquals, test.manifestPkgs) } - - // Find the log output of this test by trimming the suite output - // until we find the last occurrence of the summary. - testLogs := strings.Split(c.GetTestLog(), logMarker) - logOutput := testLogs[len(testLogs)-1] - - // Assert log output. - if test.logOutput != "" { - c.Assert(logOutput, Matches, test.logOutput) - } else { - // No warnings emitted. - c.Assert(logOutput, Not(Matches), "(?s).*Warning.*") - } } } From a902099e6b403365c98b34b520d960880471039b Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 09:54:10 +0100 Subject: [PATCH 48/65] fix: apply PR suggestions --- internal/slicer/slicer.go | 26 ++++++++++++++++++++- internal/slicer/slicer_test.go | 41 ++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index d27552d3..a81563ee 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -400,9 +400,33 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes entry := report.Entries[path] srcPath := filepath.Clean(filepath.Join(tempDir, path)) dstPath := filepath.Clean(filepath.Join(targetDir, path)) - if err := os.MkdirAll(filepath.Dir(dstPath), 0o755); err != nil { + + // Create parent directories, removing any file on the way up. + var mkParent func(path string) error + mkParent = func(path string) error { + parent := filepath.Dir(path) + err := os.MkdirAll(parent, 0o755) + if err != nil { + e, ok := err.(*os.PathError) + if ok && errors.Is(e.Unwrap(), syscall.ENOTDIR) { + err := os.Remove(parent) + if err != nil { + return err + } + err = os.MkdirAll(parent, 0o755) + if err != nil { + return mkParent(parent) + } + return nil + } + return err + } + return nil + } + if err := mkParent(dstPath); err != nil { return err } + var err error switch entry.Mode & fs.ModeType { case 0, fs.ModeSymlink: diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index a4df83e9..452ecfc3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2167,21 +2167,21 @@ func runSlicerTests(s *S, c *C, tests []slicerTest) { } type slicerRecutTest struct { - summary string - arch string - release map[string]string - pkgs []*testutil.TestPackage - cutSlices []setup.SliceKey - recutSlices []setup.SliceKey - hackopt func(c *C, opts *slicer.RunOptions) + summary string + arch string + release map[string]string + pkgs []*testutil.TestPackage + cutSlices []setup.SliceKey + recutSlices []setup.SliceKey + hackopt func(c *C, opts *slicer.RunOptions) // Modifies the filesystem built after the first execution and before the // second one. alterFilesystem func(c *C, targetDir string) - filesystem map[string]string - manifestPaths map[string]string - manifestPkgs map[string]string - logOutput string - error string + filesystem map[string]string + manifestPaths map[string]string + manifestPkgs map[string]string + logOutput string + error string } var slicerRecutTests = []slicerRecutTest{{ @@ -2343,15 +2343,12 @@ var slicerRecutTests = []slicerRecutTest{{ `, }, alterFilesystem: func(c *C, targetDir string) { - err := os.WriteFile(filepath.Join(targetDir, "bar"), []byte("data"), 0o644) - c.Assert(err, IsNil) linkPath := filepath.Join(targetDir, "foo") - err = os.Symlink("bar", linkPath) + err := os.Symlink("bar", linkPath) c.Assert(err, IsNil) }, filesystem: map[string]string{ "/dir/": "dir 0755", - "/bar": "file 0644 3a6eb079", "/baz": "file 0644 3a6eb079", "/foo": "symlink baz", }, @@ -2361,7 +2358,7 @@ var slicerRecutTests = []slicerRecutTest{{ "/foo": "symlink baz {test-package_slice2}", }, }, { - summary: "Upgrade fails when parent is a file", + summary: "Upgrade removes content to create parent dirs", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ @@ -2382,7 +2379,13 @@ var slicerRecutTests = []slicerRecutTest{{ err = os.WriteFile(path, []byte("data"), 0o644) c.Assert(err, IsNil) }, - error: `mkdir .*: not a directory`, + filesystem: map[string]string{ + "/dir/": "dir 0755", + "/dir/file": "file 0644 3a6eb079", + }, + manifestPaths: map[string]string{ + "/dir/file": "file 0644 3a6eb079 {test-package_slice1}", + }, }, { summary: "Upgrade removes content whith unmatching type", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, @@ -2401,7 +2404,7 @@ var slicerRecutTests = []slicerRecutTest{{ filePath := filepath.Join(targetDir, "file") err := os.Remove(filePath) c.Assert(err, IsNil) - err = os.MkdirAll(filePath, 0o755) + err = os.Mkdir(filePath, 0o755) c.Assert(err, IsNil) dirPath := filepath.Join(targetDir, "a-dir") err = os.Remove(dirPath) From 08b3edd678fdcd9c1d9174e760fc66995d0bf32d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 10:16:54 +0100 Subject: [PATCH 49/65] test: improve regex precision --- internal/slicer/slicer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 452ecfc3..a56a31b3 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2672,7 +2672,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ writeManifest(c, targetDir, manifestPathA, slice, "hash1") writeManifest(c, targetDir, manifestPathB, slice, "hash2") }, - error: `cannot select a manifest: ".*" and ".*" are inconsistent`, + error: `cannot select a manifest: "/chisel-a/manifest.wall" and "/chisel-b/manifest.wall" are inconsistent`, }, { summary: "Invalid manifest data returns error", build: func() *setup.Release { From cf90f0dfa04c2699256d77359792eed804f79c8b Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 11:12:15 +0100 Subject: [PATCH 50/65] refactor: extract and simplify mkParentAll --- internal/slicer/slicer.go | 44 +++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index a81563ee..7e104070 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -402,28 +402,7 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes dstPath := filepath.Clean(filepath.Join(targetDir, path)) // Create parent directories, removing any file on the way up. - var mkParent func(path string) error - mkParent = func(path string) error { - parent := filepath.Dir(path) - err := os.MkdirAll(parent, 0o755) - if err != nil { - e, ok := err.(*os.PathError) - if ok && errors.Is(e.Unwrap(), syscall.ENOTDIR) { - err := os.Remove(parent) - if err != nil { - return err - } - err = os.MkdirAll(parent, 0o755) - if err != nil { - return mkParent(parent) - } - return nil - } - return err - } - return nil - } - if err := mkParent(dstPath); err != nil { + if err := mkParentAll(dstPath); err != nil { return err } @@ -467,6 +446,27 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes return nil } +func mkParentAll(path string) error { + parent := filepath.Dir(path) + err := os.MkdirAll(parent, 0o755) + if err == nil { + return nil + } + e, ok := err.(*os.PathError) + if !ok || !errors.Is(e.Unwrap(), syscall.ENOTDIR) { + return err + } + err = os.Remove(parent) + if err != nil { + return err + } + err = os.MkdirAll(parent, 0o755) + if err != nil { + return mkParentAll(parent) + } + return nil +} + func upgradeDir(path string, entry manifestutil.ReportEntry) error { err := os.Mkdir(path, entry.Mode) if err != nil { From 46f872c972f008c79fe776f717947944f4013b09 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 15:57:14 +0100 Subject: [PATCH 51/65] fix: Error when only invalid manifests found --- internal/slicer/slicer.go | 5 +++++ internal/slicer/slicer_test.go | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 7e104070..ec8ab5bf 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -713,6 +713,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma var selected *manifest.Manifest var selectedHash string var selectedPath string + foundUnknownSchema := false for _, mfestPath := range manifestPaths { mfestFullPath := path.Join(targetDir, mfestPath) f, err := os.Open(mfestFullPath) @@ -731,6 +732,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma mfest, err := manifest.Read(r) if err != nil { if errors.Is(err, manifest.ErrUnknownSchema) { + foundUnknownSchema = true // Ignore manifests with unknown (potentially future) schema versions. continue } @@ -753,6 +755,9 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma return nil, fmt.Errorf("cannot select a manifest: %q and %q are inconsistent", selectedPath, mfestPath) } } + if foundUnknownSchema && selected == nil { + return nil, fmt.Errorf("cannot select a manifest: manifest(s) found use unknown schema") + } return selected, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index a56a31b3..ea2ee967 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2629,7 +2629,19 @@ var selectValidManifestTests = []selectValidManifestTest{{ }, noMatch: true, }, { - summary: "Unknown schema error ignored", + summary: "Unknown schema error ignored when other valid found", + build: func() *setup.Release { + return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") + }, + setup: func(c *C, targetDir string, release *setup.Release) { + manifestPathA := manifestPathForDir("/chisel-a/**") + manifestPathB := manifestPathForDir("/chisel-b/**") + slice := release.Packages["test-package"].Slices["manifest"] + writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeInvalidSchemaManifest(c, targetDir, manifestPathB) + }, +}, { + summary: "Unknown schema error raised when no other valid found", build: func() *setup.Release { return buildReleaseWithManifestDirs("/chisel/**") }, @@ -2637,7 +2649,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ manifestPath := manifestPathForDir("/chisel/**") writeInvalidSchemaManifest(c, targetDir, manifestPath) }, - noMatch: true, + error: `cannot select a manifest: manifest\(s\) found use unknown schema`, }, { summary: "Valid manifest selected", build: func() *setup.Release { From 2052ee5ffe082f6cb91271b6df0dd99d96721e5f Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 15:58:16 +0100 Subject: [PATCH 52/65] fix: lint --- internal/slicer/slicer_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index ea2ee967..d19441b7 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2180,7 +2180,6 @@ type slicerRecutTest struct { filesystem map[string]string manifestPaths map[string]string manifestPkgs map[string]string - logOutput string error string } From 0fbc47ad6316978bfab76fad3cb1da1d8771f677 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 13 Feb 2026 15:58:41 +0100 Subject: [PATCH 53/65] fix: wrap add context on upgrade errors --- internal/slicer/slicer.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index ec8ab5bf..e9bc859e 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -403,15 +403,21 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes // Create parent directories, removing any file on the way up. if err := mkParentAll(dstPath); err != nil { - return err + return fmt.Errorf("cannot create parent directory for %q: %s", path, err) } var err error switch entry.Mode & fs.ModeType { case 0, fs.ModeSymlink: err = upgradeFile(srcPath, dstPath) + if err != nil { + err = fmt.Errorf("cannot upgrade file at %q: %s", path, err) + } case fs.ModeDir: err = upgradeDir(dstPath, entry) + if err != nil { + err = fmt.Errorf("cannot upgrade directory at %q: %s", path, err) + } default: err = fmt.Errorf("unsupported file type: %s", path) } From 30f1c3218d9d076368a702827daa1d69624f262c Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 23 Feb 2026 14:19:08 +0100 Subject: [PATCH 54/65] style: improve error message --- internal/slicer/slicer.go | 2 +- internal/slicer/slicer_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index e9bc859e..003e6998 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -762,7 +762,7 @@ func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Ma } } if foundUnknownSchema && selected == nil { - return nil, fmt.Errorf("cannot select a manifest: manifest(s) found use unknown schema") + return nil, fmt.Errorf("cannot select a manifest: all manifests found use unknown schema") } return selected, nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index d19441b7..2c324773 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2648,7 +2648,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ manifestPath := manifestPathForDir("/chisel/**") writeInvalidSchemaManifest(c, targetDir, manifestPath) }, - error: `cannot select a manifest: manifest\(s\) found use unknown schema`, + error: `cannot select a manifest: all manifests found use unknown schema`, }, { summary: "Valid manifest selected", build: func() *setup.Release { From a675ad8e0b6ab0e6e61c45a668e33a523b4f4be5 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Mon, 23 Feb 2026 14:24:13 +0100 Subject: [PATCH 55/65] tests: previous package is in the new manifest --- tests/recut/chisel-releases/slices/hello.yaml | 13 +++++++++++++ tests/recut/task.yaml | 7 ++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/recut/chisel-releases/slices/hello.yaml diff --git a/tests/recut/chisel-releases/slices/hello.yaml b/tests/recut/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/recut/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index b87119c3..170fbe68 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -18,10 +18,11 @@ execute: | # First cut generates manifest and installs slice-a. chisel cut --release ./chisel-releases/ \ --root ${ROOTFS} \ - base-files_slice-a base-files_manifest + base-files_slice-a hello_bins base-files_manifest test -s ${ROOTFS}/etc/debian_version test -s ${ROOTFS}/etc/foo + test -s ${ROOTFS}/usr/bin/hello cat ${ROOTFS}/etc/foo | grep "bar" test -f ${ROOTFS}/chisel/manifest.wall @@ -37,3 +38,7 @@ execute: | test -s ${ROOTFS}/etc/issue cat ${ROOTFS}/etc/foo | grep "qux" test -f ${ROOTFS}/chisel/manifest.wall + zstd -d ${ROOTFS}/chisel/manifest.wall -o ${ROOTFS}/chisel/manifest.jsonwall + packages=$(jq -r 'select(.kind == "package") | .name' ${ROOTFS}/chisel/manifest.jsonwall) + echo $packages | MATCH "base-files" + echo $packages | MATCH "hello" From 558b8276f73f9f0e75ea6eff7dd5523c614f7652 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 25 Feb 2026 09:07:58 +0100 Subject: [PATCH 56/65] fix: properly replicate parent dirs when upgrading --- internal/slicer/slicer.go | 61 +++++++++++++++++++++------------- internal/slicer/slicer_test.go | 41 +++++++++++++++-------- 2 files changed, 64 insertions(+), 38 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 003e6998..9f5b0b6b 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -401,8 +401,7 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes srcPath := filepath.Clean(filepath.Join(tempDir, path)) dstPath := filepath.Clean(filepath.Join(targetDir, path)) - // Create parent directories, removing any file on the way up. - if err := mkParentAll(dstPath); err != nil { + if err := upgradeParentDirs(tempDir, targetDir, path); err != nil { return fmt.Errorf("cannot create parent directory for %q: %s", path, err) } @@ -414,7 +413,7 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes err = fmt.Errorf("cannot upgrade file at %q: %s", path, err) } case fs.ModeDir: - err = upgradeDir(dstPath, entry) + err = upgradeDir(dstPath, entry.Mode) if err != nil { err = fmt.Errorf("cannot upgrade directory at %q: %s", path, err) } @@ -452,29 +451,43 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes return nil } -func mkParentAll(path string) error { - parent := filepath.Dir(path) - err := os.MkdirAll(parent, 0o755) - if err == nil { - return nil - } - e, ok := err.(*os.PathError) - if !ok || !errors.Is(e.Unwrap(), syscall.ENOTDIR) { - return err - } - err = os.Remove(parent) - if err != nil { - return err - } - err = os.MkdirAll(parent, 0o755) - if err != nil { - return mkParentAll(parent) +// upgradeParentDirs replicates the parent directories of targetPath in dstRoot, +// removing any non-directory on the way. +func upgradeParentDirs(srcRoot string, dstRoot string, targetPath string) error { + parents := parentDirs(targetPath) + for _, path := range parents { + if path == "/" { + continue + } + srcPath := filepath.Clean(filepath.Join(srcRoot, path)) + srcInfo, err := os.Stat(srcPath) + if err != nil { + return err + } + dstPath := filepath.Clean(filepath.Join(dstRoot, path)) + err = upgradeDir(dstPath, srcInfo.Mode()) + if err != nil { + return err + } } return nil } -func upgradeDir(path string, entry manifestutil.ReportEntry) error { - err := os.Mkdir(path, entry.Mode) +func parentDirs(path string) []string { + path = filepath.Clean(path) + parents := make([]string, strings.Count(path, "/")) + count := 0 + for i, c := range path { + if c == '/' { + parents[count] = path[:i+1] + count++ + } + } + return parents +} + +func upgradeDir(path string, mode fs.FileMode) error { + err := os.Mkdir(path, mode) if err != nil { if !os.IsExist(err) { return err @@ -484,7 +497,7 @@ func upgradeDir(path string, entry manifestutil.ReportEntry) error { return err } if fileinfo.IsDir() { - return os.Chmod(path, entry.Mode) + return os.Chmod(path, mode) } // Path is a regular file or symlink, remove it. err = os.Remove(path) @@ -492,7 +505,7 @@ func upgradeDir(path string, entry manifestutil.ReportEntry) error { return err } - return os.Mkdir(path, entry.Mode) + return os.Mkdir(path, mode) } return nil } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 2c324773..0537f9a1 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2199,21 +2199,26 @@ var slicerRecutTests = []slicerRecutTest{{ slice2: contents: /dir/other-file: + /parent/permissions/file: `, }, filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/file": "file 0644 cc55e2ec", - "/dir/file-copy": "file 0644 cc55e2ec", - "/dir/other-file": "file 0644 63d5dd49", - "/other-dir/": "dir 0755", - "/other-dir/file": "symlink ../dir/file", + "/dir/": "dir 0755", + "/dir/file": "file 0644 cc55e2ec", + "/dir/file-copy": "file 0644 cc55e2ec", + "/dir/other-file": "file 0644 63d5dd49", + "/other-dir/": "dir 0755", + "/other-dir/file": "symlink ../dir/file", + "/parent/": "dir 01777", + "/parent/permissions/": "dir 0764", + "/parent/permissions/file": "file 0755 722c14b3", }, manifestPaths: map[string]string{ - "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", - "/dir/file-copy": "file 0644 cc55e2ec {test-package_slice1}", - "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", - "/other-dir/file": "symlink ../dir/file {test-package_slice1}", + "/dir/file": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/file-copy": "file 0644 cc55e2ec {test-package_slice1}", + "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", + "/other-dir/file": "symlink ../dir/file {test-package_slice1}", + "/parent/permissions/file": "file 0755 722c14b3 {test-package_slice2}", }, }, { summary: "Upgrade removes obsolete paths when selection shrinks", @@ -2357,7 +2362,7 @@ var slicerRecutTests = []slicerRecutTest{{ "/foo": "symlink baz {test-package_slice2}", }, }, { - summary: "Upgrade removes content to create parent dirs", + summary: "Upgrade removes and change mode of content to create parent dirs", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ @@ -2369,6 +2374,7 @@ var slicerRecutTests = []slicerRecutTest{{ /dir/file: {text: data} slice2: contents: + /parent/permissions/file: `, }, alterFilesystem: func(c *C, targetDir string) { @@ -2377,13 +2383,20 @@ var slicerRecutTests = []slicerRecutTest{{ c.Assert(err, IsNil) err = os.WriteFile(path, []byte("data"), 0o644) c.Assert(err, IsNil) + newDirPath := filepath.Join(targetDir, "parent/permissions/") + err = os.MkdirAll(newDirPath, 0o755) + c.Assert(err, IsNil) }, filesystem: map[string]string{ - "/dir/": "dir 0755", - "/dir/file": "file 0644 3a6eb079", + "/dir/": "dir 0755", + "/dir/file": "file 0644 3a6eb079", + "/parent/": "dir 01777", + "/parent/permissions/": "dir 0764", + "/parent/permissions/file": "file 0755 722c14b3", }, manifestPaths: map[string]string{ - "/dir/file": "file 0644 3a6eb079 {test-package_slice1}", + "/dir/file": "file 0644 3a6eb079 {test-package_slice1}", + "/parent/permissions/file": "file 0755 722c14b3 {test-package_slice2}", }, }, { summary: "Upgrade removes content whith unmatching type", From ab117970bdfcae4b41f0ca2baaf4acd09eed1d3d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 25 Feb 2026 09:55:16 +0100 Subject: [PATCH 57/65] tests: Verify packages in manifest after recut --- internal/slicer/slicer_test.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 0537f9a1..01a470bd 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2173,7 +2173,6 @@ type slicerRecutTest struct { pkgs []*testutil.TestPackage cutSlices []setup.SliceKey recutSlices []setup.SliceKey - hackopt func(c *C, opts *slicer.RunOptions) // Modifies the filesystem built after the first execution and before the // second one. alterFilesystem func(c *C, targetDir string) @@ -2186,7 +2185,20 @@ type slicerRecutTest struct { var slicerRecutTests = []slicerRecutTest{{ summary: "Basic upgrade", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, - recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, + recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}, {"other-package", "slice1"}}, + pkgs: []*testutil.TestPackage{{ + Name: "test-package", + Hash: "h1", + Version: "v1", + Arch: "a1", + Data: testutil.PackageData["test-package"], + }, { + Name: "other-package", + Hash: "h2", + Version: "v2", + Arch: "a2", + Data: testutil.PackageData["other-package"], + }}, release: map[string]string{ "slices/mydir/test-package.yaml": ` package: test-package @@ -2201,12 +2213,20 @@ var slicerRecutTests = []slicerRecutTest{{ /dir/other-file: /parent/permissions/file: `, + "slices/mydir/other-package.yaml": ` + package: other-package + slices: + slice1: + contents: + /file: + `, }, filesystem: map[string]string{ "/dir/": "dir 0755", "/dir/file": "file 0644 cc55e2ec", "/dir/file-copy": "file 0644 cc55e2ec", "/dir/other-file": "file 0644 63d5dd49", + "/file": "file 0644 fc02ca0e", "/other-dir/": "dir 0755", "/other-dir/file": "symlink ../dir/file", "/parent/": "dir 01777", @@ -2219,6 +2239,11 @@ var slicerRecutTests = []slicerRecutTest{{ "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", "/other-dir/file": "symlink ../dir/file {test-package_slice1}", "/parent/permissions/file": "file 0755 722c14b3 {test-package_slice2}", + "/file": "file 0644 fc02ca0e {other-package_slice1}", + }, + manifestPkgs: map[string]string{ + "test-package": "test-package v1 a1 h1", + "other-package": "other-package v2 a2 h2", }, }, { summary: "Upgrade removes obsolete paths when selection shrinks", @@ -2554,9 +2579,6 @@ func (s *S) TestRunRecut(c *C) { Archives: archives, TargetDir: targetDir, } - if test.hackopt != nil { - test.hackopt(c, &options) - } // First run. err = slicer.Run(&options) c.Assert(err, IsNil) From 9ff9bccb88d5ed1e70f179b84b9aaa043e63c028 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Wed, 25 Feb 2026 11:51:11 +0100 Subject: [PATCH 58/65] tests: check exactly the package list --- tests/recut/task.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/recut/task.yaml b/tests/recut/task.yaml index 170fbe68..fc426ef5 100644 --- a/tests/recut/task.yaml +++ b/tests/recut/task.yaml @@ -40,5 +40,9 @@ execute: | test -f ${ROOTFS}/chisel/manifest.wall zstd -d ${ROOTFS}/chisel/manifest.wall -o ${ROOTFS}/chisel/manifest.jsonwall packages=$(jq -r 'select(.kind == "package") | .name' ${ROOTFS}/chisel/manifest.jsonwall) - echo $packages | MATCH "base-files" - echo $packages | MATCH "hello" + expected=$(cat < Date: Thu, 26 Feb 2026 11:47:44 +0100 Subject: [PATCH 59/65] fix: nitpicks and typos --- internal/slicer/slicer_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 01a470bd..b472c55b 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2269,7 +2269,7 @@ var slicerRecutTests = []slicerRecutTest{{ "/dir/other-file": "file 0644 63d5dd49 {test-package_slice2}", }, }, { - summary: "Upgrade restores modified content and mode", + summary: "Upgrade overrides modified content and mode", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}}, release: map[string]string{ @@ -2339,7 +2339,7 @@ var slicerRecutTests = []slicerRecutTest{{ `, }, alterFilesystem: func(c *C, targetDir string) { - err := os.MkdirAll(filepath.Join(targetDir, "other-dir"), 0o775) + err := os.Mkdir(filepath.Join(targetDir, "other-dir"), 0o775) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(targetDir, "dir", "file"), []byte("data"), 0o644) c.Assert(err, IsNil) @@ -2387,7 +2387,7 @@ var slicerRecutTests = []slicerRecutTest{{ "/foo": "symlink baz {test-package_slice2}", }, }, { - summary: "Upgrade removes and change mode of content to create parent dirs", + summary: "Upgrade removes and changes mode of content to create parent dirs", cutSlices: []setup.SliceKey{{"test-package", "slice1"}}, recutSlices: []setup.SliceKey{{"test-package", "slice1"}, {"test-package", "slice2"}}, release: map[string]string{ @@ -2671,7 +2671,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") slice := release.Packages["test-package"].Slices["manifest"] - writeManifest(c, targetDir, manifestPathA, slice, "hash1") + writeSampleManifest(c, targetDir, manifestPathA, slice, "hash1") writeInvalidSchemaManifest(c, targetDir, manifestPathB) }, }, { @@ -2692,7 +2692,7 @@ var selectValidManifestTests = []selectValidManifestTest{{ setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := manifestPathForDir("/chisel/**") slice := release.Packages["test-package"].Slices["manifest"] - writeManifest(c, targetDir, manifestPath, slice, "hash1") + writeSampleManifest(c, targetDir, manifestPath, slice, "hash1") }, }, { summary: "Two consistent manifests are accepted", @@ -2703,8 +2703,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") slice := release.Packages["test-package"].Slices["manifest"] - writeManifest(c, targetDir, manifestPathA, slice, "hash1") - writeManifest(c, targetDir, manifestPathB, slice, "hash1") + writeSampleManifest(c, targetDir, manifestPathA, slice, "hash1") + writeSampleManifest(c, targetDir, manifestPathB, slice, "hash1") }, }, { summary: "Inconsistent manifests with same schema are rejected", @@ -2715,8 +2715,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") slice := release.Packages["test-package"].Slices["manifest"] - writeManifest(c, targetDir, manifestPathA, slice, "hash1") - writeManifest(c, targetDir, manifestPathB, slice, "hash2") + writeSampleManifest(c, targetDir, manifestPathA, slice, "hash1") + writeSampleManifest(c, targetDir, manifestPathB, slice, "hash2") }, error: `cannot select a manifest: "/chisel-a/manifest.wall" and "/chisel-b/manifest.wall" are inconsistent`, }, { @@ -2869,7 +2869,7 @@ func readManifest(c *C, targetDir, manifestPath string) *manifest.Manifest { return mfest } -func writeManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { +func writeSampleManifest(c *C, targetDir, manifestPath string, slice *setup.Slice, hash string) { mfestPath := filepath.Join(targetDir, manifestPath) err := os.MkdirAll(filepath.Dir(mfestPath), 0o755) c.Assert(err, IsNil) @@ -2927,7 +2927,7 @@ func writeInvalidSchemaManifest(c *C, targetDir, manifestPath string) { c.Assert(err, IsNil) zw, err := zstd.NewWriter(f) c.Assert(err, IsNil) - dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: "9.9"}) + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{Schema: "invalid"}) _, err = dbw.WriteTo(zw) c.Assert(err, IsNil) c.Assert(zw.Close(), IsNil) From 78efc4583d4d738a651d656bdf79de2d7a03e808 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 26 Feb 2026 11:48:19 +0100 Subject: [PATCH 60/65] docs: Expose reasoning behind complex dir creation --- internal/slicer/slicer.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9f5b0b6b..62ad81f3 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -392,7 +392,7 @@ func install(options *RunOptions) (*manifestutil.Report, error) { return report, nil } -// upgrade upgrades content in targetDir with content in tempDir. +// upgrade upgrades content in targetDir with content from tempDir. func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfest *manifest.Manifest) error { logf("Upgrading content...") paths := slices.Sorted(maps.Keys(report.Entries)) @@ -401,6 +401,11 @@ func upgrade(targetDir string, tempDir string, report *manifestutil.Report, mfes srcPath := filepath.Clean(filepath.Join(tempDir, path)) dstPath := filepath.Clean(filepath.Join(targetDir, path)) + // When extracting the content, a great care is taken to create parent + // directories respecting the tarball permissions. However the current + // approach may not record some of these parents in the report. Make sure + // to create consistent directories in the targetDir to sustain the same + // guarantees as a normal cut. if err := upgradeParentDirs(tempDir, targetDir, path); err != nil { return fmt.Errorf("cannot create parent directory for %q: %s", path, err) } From b8d695f557336c41c18d17ecf825cf692764e6be Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 26 Feb 2026 11:52:12 +0100 Subject: [PATCH 61/65] refactor: clarify test intent --- internal/slicer/slicer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index b472c55b..918913f8 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2307,7 +2307,7 @@ var slicerRecutTests = []slicerRecutTest{{ `, }, alterFilesystem: func(c *C, targetDir string) { - err := os.MkdirAll(filepath.Join(targetDir, "extra"), 0o755) + err := os.Mkdir(filepath.Join(targetDir, "extra"), 0o755) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(targetDir, "extra", "untracked"), []byte("data"), 0o644) c.Assert(err, IsNil) From b81166111d2bffc8594d7877f3afa44530706b8e Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Thu, 26 Feb 2026 13:28:58 +0100 Subject: [PATCH 62/65] tests: test equivalence between a single and 2 cuts --- .../chisel-releases/chisel.yaml | 49 +++++++++++++++++++ .../chisel-releases/slices/base-files.yaml | 13 +++++ .../chisel-releases/slices/hello.yaml | 13 +++++ tests/recut-equivalence/task.yaml | 35 +++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 tests/recut-equivalence/chisel-releases/chisel.yaml create mode 100644 tests/recut-equivalence/chisel-releases/slices/base-files.yaml create mode 100644 tests/recut-equivalence/chisel-releases/slices/hello.yaml create mode 100644 tests/recut-equivalence/task.yaml diff --git a/tests/recut-equivalence/chisel-releases/chisel.yaml b/tests/recut-equivalence/chisel-releases/chisel.yaml new file mode 100644 index 00000000..c137ff40 --- /dev/null +++ b/tests/recut-equivalence/chisel-releases/chisel.yaml @@ -0,0 +1,49 @@ +format: v2 + + +archives: + # archive.ubuntu.com/ubuntu/ (amd64, i386) + # ports.ubuntu.com/ubuntu-ports/ (other arch) + ubuntu: + priority: 10 + version: 24.04 + components: [main, universe] + suites: [noble, noble-security, noble-updates] + public-keys: [ubuntu-archive-key-2018] + + +public-keys: + # Ubuntu Archive Automatic Signing Key (2018) + # rsa4096/f6ecb3762474eda9d21b7022871920d1991bc93c 2018-09-17T15:01:46Z + ubuntu-archive-key-2018: + id: "871920D1991BC93C" + armor: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFufwdoBEADv/Gxytx/LcSXYuM0MwKojbBye81s0G1nEx+lz6VAUpIUZnbkq + dXBHC+dwrGS/CeeLuAjPRLU8AoxE/jjvZVp8xFGEWHYdklqXGZ/gJfP5d3fIUBtZ + HZEJl8B8m9pMHf/AQQdsC+YzizSG5t5Mhnotw044LXtdEEkx2t6Jz0OGrh+5Ioxq + X7pZiq6Cv19BohaUioKMdp7ES6RYfN7ol6HSLFlrMXtVfh/ijpN9j3ZhVGVeRC8k + KHQsJ5PkIbmvxBiUh7SJmfZUx0IQhNMaDHXfdZAGNtnhzzNReb1FqNLSVkrS/Pns + AQzMhG1BDm2VOSF64jebKXffFqM5LXRQTeqTLsjUbbrqR6s/GCO8UF7jfUj6I7ta + LygmsHO/JD4jpKRC0gbpUBfaiJyLvuepx3kWoqL3sN0LhlMI80+fA7GTvoOx4tpq + VlzlE6TajYu+jfW3QpOFS5ewEMdL26hzxsZg/geZvTbArcP+OsJKRmhv4kNo6Ayd + yHQ/3ZV/f3X9mT3/SPLbJaumkgp3Yzd6t5PeBu+ZQk/mN5WNNuaihNEV7llb1Zhv + Y0Fxu9BVd/BNl0rzuxp3rIinB2TX2SCg7wE5xXkwXuQ/2eTDE0v0HlGntkuZjGow + DZkxHZQSxZVOzdZCRVaX/WEFLpKa2AQpw5RJrQ4oZ/OfifXyJzP27o03wQARAQAB + tEJVYnVudHUgQXJjaGl2ZSBBdXRvbWF0aWMgU2lnbmluZyBLZXkgKDIwMTgpIDxm + dHBtYXN0ZXJAdWJ1bnR1LmNvbT6JAjgEEwEKACIFAlufwdoCGwMGCwkIBwMCBhUI + AgkKCwQWAgMBAh4BAheAAAoJEIcZINGZG8k8LHMQAKS2cnxz/5WaoCOWArf5g6UH + beOCgc5DBm0hCuFDZWWv427aGei3CPuLw0DGLCXZdyc5dqE8mvjMlOmmAKKlj1uG + g3TYCbQWjWPeMnBPZbkFgkZoXJ7/6CB7bWRht1sHzpt1LTZ+SYDwOwJ68QRp7DRa + Zl9Y6QiUbeuhq2DUcTofVbBxbhrckN4ZteLvm+/nG9m/ciopc66LwRdkxqfJ32Cy + q+1TS5VaIJDG7DWziG+Kbu6qCDM4QNlg3LH7p14CrRxAbc4lvohRgsV4eQqsIcdF + kuVY5HPPj2K8TqpY6STe8Gh0aprG1RV8ZKay3KSMpnyV1fAKn4fM9byiLzQAovC0 + LZ9MMMsrAS/45AvC3IEKSShjLFn1X1dRCiO6/7jmZEoZtAp53hkf8SMBsi78hVNr + BumZwfIdBA1v22+LY4xQK8q4XCoRcA9G+pvzU9YVW7cRnDZZGl0uwOw7z9PkQBF5 + KFKjWDz4fCk+K6+YtGpovGKekGBb8I7EA6UpvPgqA/QdI0t1IBP0N06RQcs1fUaA + QEtz6DGy5zkRhR4pGSZn+dFET7PdAjEK84y7BdY4t+U1jcSIvBj0F2B7LwRL7xGp + SpIKi/ekAXLs117bvFHaCvmUYN7JVp1GMmVFxhIdx6CFm3fxG8QjNb5tere/YqK+ + uOgcXny1UlwtCUzlrSaP + =9AdM + -----END PGP PUBLIC KEY BLOCK----- diff --git a/tests/recut-equivalence/chisel-releases/slices/base-files.yaml b/tests/recut-equivalence/chisel-releases/slices/base-files.yaml new file mode 100644 index 00000000..84c30016 --- /dev/null +++ b/tests/recut-equivalence/chisel-releases/slices/base-files.yaml @@ -0,0 +1,13 @@ +package: base-files +slices: + slice-a: + contents: + /etc/debian_version: + /etc/foo: + text: bar + slice-b: + contents: + /etc/issue: + manifest: + contents: + /chisel/**: {generate: manifest} diff --git a/tests/recut-equivalence/chisel-releases/slices/hello.yaml b/tests/recut-equivalence/chisel-releases/slices/hello.yaml new file mode 100644 index 00000000..7bd1a393 --- /dev/null +++ b/tests/recut-equivalence/chisel-releases/slices/hello.yaml @@ -0,0 +1,13 @@ +package: hello + +essential: + - hello_copyright + +slices: + bins: + contents: + /usr/bin/hello: + + copyright: + contents: + /usr/share/doc/hello/copyright: diff --git a/tests/recut-equivalence/task.yaml b/tests/recut-equivalence/task.yaml new file mode 100644 index 00000000..764dd9bf --- /dev/null +++ b/tests/recut-equivalence/task.yaml @@ -0,0 +1,35 @@ +summary: Recut produce the exact same result as two consecutive cuts. + +variants: + - noble + +environment: + ROOTFS: rootfs + ROOTFS_RECUT: rootfs-recut + +execute: | + mkdir -p ${ROOTFS} ${ROOTFS_RECUT} + + # TODO: remove this env var when final upgrade strategy is in place. + export CHISEL_RECUT_EXPERIMENTAL=1 + + # First cut single cut + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS} \ + hello_bins \ + base-files_manifest \ + base-files_slice-a \ + base-files_slice-b + + # Cut a rootfs twice. + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS_RECUT} \ + hello_bins \ + base-files_manifest \ + base-files_slice-a + chisel cut --release ./chisel-releases/ \ + --root ${ROOTFS_RECUT} \ + base-files_slice-b + + # Compares both rootfs. + diff -q -r ${ROOTFS} ${ROOTFS_RECUT} From acebd11a62664ad3c27a0b226087e6b5808b3f41 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 27 Feb 2026 10:28:51 +0100 Subject: [PATCH 63/65] docs: outlines important aspects of the overall logic --- internal/slicer/slicer.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 62ad81f3..075118aa 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -718,8 +718,18 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel return pkgArchive, nil } -// SelectValidManifest returns, if found, a valid manifest. Consistency with -// other manifests is verified so the selection is deterministic. +// SelectValidManifest returns, if found, a valid manifest. +// +// Not finding any manifest (valid or not) means the targetDir cannot be +// considered as previously produced by Chisel for the given release. +// +// Finding only manifests with unknown schema version means the targetDir may +// have been produced by Chisel, but possibly by a future/incompatible one. +// Chisel cannot safely proceed and so errors out. +// +// Finding multiple manifests, with at least one valid means Chisel can proceeds, +// ignoring unknown ones. Consistency between valid manifests is verified, +// ensuring a deterministic selection. func SelectValidManifest(targetDir string, release *setup.Release) (*manifest.Manifest, error) { targetDir = filepath.Clean(targetDir) if !filepath.IsAbs(targetDir) { From 6e6b09e07d1fffc0913dedd9b128cd0174a1a860 Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 27 Feb 2026 11:36:21 +0100 Subject: [PATCH 64/65] tests: simplify TestSelectValidManifest --- internal/slicer/slicer_test.go | 107 ++++++++++++++------------------- 1 file changed, 44 insertions(+), 63 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 918913f8..daeec856 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2643,30 +2643,23 @@ func (s *S) TestRunRecut(c *C) { } type selectValidManifestTest struct { - summary string - build func() *setup.Release - setup func(c *C, targetDir string, release *setup.Release) - noMatch bool - error string + summary string + setup func(c *C, targetDir string, release *setup.Release) + manifestDirs []string + noMatch bool + error string } var selectValidManifestTests = []selectValidManifestTest{{ summary: "No manifest paths in release", - build: func() *setup.Release { - return &setup.Release{Packages: map[string]*setup.Package{}} - }, noMatch: true, }, { - summary: "Manifest path missing in target", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, - noMatch: true, + summary: "Manifest path missing in target", + manifestDirs: []string{"/chisel/**"}, + noMatch: true, }, { - summary: "Unknown schema error ignored when other valid found", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") - }, + summary: "Unknown schema error ignored when other valid found", + manifestDirs: []string{"/chisel-a/**", "/chisel-b/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") @@ -2675,30 +2668,24 @@ var selectValidManifestTests = []selectValidManifestTest{{ writeInvalidSchemaManifest(c, targetDir, manifestPathB) }, }, { - summary: "Unknown schema error raised when no other valid found", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, + summary: "Unknown schema error raised when no other valid found", + manifestDirs: []string{"/chisel/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := manifestPathForDir("/chisel/**") writeInvalidSchemaManifest(c, targetDir, manifestPath) }, error: `cannot select a manifest: all manifests found use unknown schema`, }, { - summary: "Valid manifest selected", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, + summary: "Valid manifest selected", + manifestDirs: []string{"/chisel/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := manifestPathForDir("/chisel/**") slice := release.Packages["test-package"].Slices["manifest"] writeSampleManifest(c, targetDir, manifestPath, slice, "hash1") }, }, { - summary: "Two consistent manifests are accepted", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") - }, + summary: "Two consistent manifests are accepted", + manifestDirs: []string{"/chisel-a/**", "/chisel-b/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") @@ -2707,10 +2694,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ writeSampleManifest(c, targetDir, manifestPathB, slice, "hash1") }, }, { - summary: "Inconsistent manifests with same schema are rejected", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel-a/**", "/chisel-b/**") - }, + summary: "Inconsistent manifests with same schema are rejected", + manifestDirs: []string{"/chisel-a/**", "/chisel-b/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPathA := manifestPathForDir("/chisel-a/**") manifestPathB := manifestPathForDir("/chisel-b/**") @@ -2720,10 +2705,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ }, error: `cannot select a manifest: "/chisel-a/manifest.wall" and "/chisel-b/manifest.wall" are inconsistent`, }, { - summary: "Invalid manifest data returns error", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, + summary: "Invalid manifest data returns error", + manifestDirs: []string{"/chisel/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := filepath.Join(targetDir, manifestPathForDir("/chisel/**")) err := os.MkdirAll(filepath.Dir(manifestPath), 0o755) @@ -2733,10 +2716,8 @@ var selectValidManifestTests = []selectValidManifestTest{{ }, error: "cannot read manifest: invalid input: .*", }, { - summary: "Manifest validation error is returned", - build: func() *setup.Release { - return buildReleaseWithManifestDirs("/chisel/**") - }, + summary: "Manifest validation error is returned", + manifestDirs: []string{"/chisel/**"}, setup: func(c *C, targetDir string, release *setup.Release) { manifestPath := manifestPathForDir("/chisel/**") writeInvalidManifest(c, targetDir, manifestPath) @@ -2747,7 +2728,28 @@ var selectValidManifestTests = []selectValidManifestTest{{ func (s *S) TestSelectValidManifest(c *C) { for _, test := range selectValidManifestTests { c.Logf("Summary: %s", test.summary) - release := test.build() + packages := map[string]*setup.Package{} + if len(test.manifestDirs) > 0 { + contents := map[string]setup.PathInfo{} + for _, dir := range test.manifestDirs { + contents[dir] = setup.PathInfo{Kind: "generate", Generate: "manifest"} + } + packages = map[string]*setup.Package{ + "test-package": { + Name: "test-package", + Slices: map[string]*setup.Slice{ + "manifest": { + Package: "test-package", + Name: "manifest", + Contents: contents, + }, + }, + }, + } + } + release := &setup.Release{ + Packages: packages, + } targetDir := c.MkDir() if test.setup != nil { test.setup(c, targetDir, release) @@ -2766,27 +2768,6 @@ func (s *S) TestSelectValidManifest(c *C) { } } -func buildReleaseWithManifestDirs(dirs ...string) *setup.Release { - contents := map[string]setup.PathInfo{} - for _, dir := range dirs { - contents[dir] = setup.PathInfo{Kind: "generate", Generate: "manifest"} - } - return &setup.Release{ - Packages: map[string]*setup.Package{ - "test-package": { - Name: "test-package", - Slices: map[string]*setup.Slice{ - "manifest": { - Package: "test-package", - Name: "manifest", - Contents: contents, - }, - }, - }, - }, - } -} - func manifestPathForDir(dir string) string { base := strings.TrimSuffix(dir, "**") return path.Join(base, manifestutil.DefaultFilename) From 76483c558bf045057b7dc6c05213eb2f9535800d Mon Sep 17 00:00:00 2001 From: Paul Mars Date: Fri, 27 Feb 2026 13:53:24 +0100 Subject: [PATCH 65/65] docs: Clarify intent on testing parent dir creation --- internal/slicer/slicer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index daeec856..93732edb 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -2229,8 +2229,8 @@ var slicerRecutTests = []slicerRecutTest{{ "/file": "file 0644 fc02ca0e", "/other-dir/": "dir 0755", "/other-dir/file": "symlink ../dir/file", - "/parent/": "dir 01777", - "/parent/permissions/": "dir 0764", + "/parent/": "dir 01777", // Permissions from the tarball preserved. + "/parent/permissions/": "dir 0764", // Permissions from the tarball preserved. "/parent/permissions/file": "file 0755 722c14b3", }, manifestPaths: map[string]string{