From ab18d0231c28a3c9ee9e376f90e07c0f1d7223bb Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 14 Oct 2026 11:44:57 +0300 Subject: [PATCH 1/5] refactor(sbom): let the image walk hand back the bytes of chosen files Reading a manifest after the index is built would cost a second pass over the layers, which on registry images is mostly gzip. The walk already has every file's bytes in hand, so a caller can now name the files it wants kept. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/file_index.go | 34 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/pkg/sbom/shippedscope/file_index.go b/pkg/sbom/shippedscope/file_index.go index 93e34949cf..6892b3945a 100644 --- a/pkg/sbom/shippedscope/file_index.go +++ b/pkg/sbom/shippedscope/file_index.go @@ -2,6 +2,7 @@ package shippedscope import ( "archive/tar" + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -44,11 +45,20 @@ type FileIndex struct { // Hardlinks make the layers be read a second time, so an image whose layers are expensive // to obtain pays for that twice. func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { + index, _, err := indexImageFiles(ctx, img, nil) + + return index, err +} + +// capture names the files whose bytes the caller needs beside their digest; they are read +// once, while the file passes by, instead of costing another walk of the image. +func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string) bool) (*FileIndex, map[string][]byte, error) { index := &FileIndex{ Files: make(map[string]File), Symlinks: make(map[string]string), } hardlinks := make(map[string]string) + captured := make(map[string][]byte) rootfs := mutate.Extract(img) defer rootfs.Close() @@ -56,7 +66,7 @@ func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { reader := tar.NewReader(rootfs) for { if err := ctx.Err(); err != nil { - return nil, err + return nil, nil, err } header, err := reader.Next() @@ -64,7 +74,7 @@ func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { break } if err != nil { - return nil, fmt.Errorf("read image filesystem: %w", err) + return nil, nil, fmt.Errorf("read image filesystem: %w", err) } name := rootedPath(header.Name) @@ -75,14 +85,24 @@ func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { switch header.Typeflag { case tar.TypeReg: hasher := sha256.New() - if _, err := io.Copy(hasher, reader); err != nil { - return nil, fmt.Errorf("hash %q: %w", name, err) + var body bytes.Buffer + + var sink io.Writer = hasher + if capture != nil && capture(name) { + sink = io.MultiWriter(hasher, &body) + } + + if _, err := io.Copy(sink, reader); err != nil { + return nil, nil, fmt.Errorf("hash %q: %w", name, err) } index.Files[name] = File{ Digest: "sha256:" + hex.EncodeToString(hasher.Sum(nil)), Size: header.Size, } + if sink != io.Writer(hasher) { + captured[name] = body.Bytes() + } case tar.TypeSymlink: index.Symlinks[name] = header.Linkname case tar.TypeLink: @@ -95,17 +115,17 @@ func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { // drain a rejected path or a layer whose digest does not verify would silently // yield a partial index instead of an error. if _, err := io.Copy(io.Discard, rootfs); err != nil { - return nil, fmt.Errorf("read image filesystem: %w", err) + return nil, nil, fmt.Errorf("read image filesystem: %w", err) } remaining, err := resolveHardlinksWithinLayers(ctx, img, index, hardlinks) if err != nil { - return nil, err + return nil, nil, err } resolveHardlinks(ctx, index, remaining) - return index, nil + return index, captured, nil } // A layer tar spells a hardlink as a regular file plus a link entry beside it, so the bytes From 091453689a33369774907cc8946435bfeaa5a150 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 14 Oct 2026 11:45:03 +0300 Subject: [PATCH 2/5] feat(sbom): collect the files each installed component ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CollectFilesets walks an image once and answers, per os-pm package, Python distribution, npm package and Lua rock, which files are that component's. Each ecosystem draws its own boundary from the manifest it writes — RECORD, package.json, rock_manifest, pm's files.json — never from a directory, so namespace packages, nested node_modules copies and rocks spread over shared trees come out right. Digests always come from the image, not from the manifest: a file an install hook rewrote ships with its current content. An image installed by a pm too old to write files.json yields no os-pm filesets and a warning rather than a failed build. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/fileset.go | 89 +++++++ pkg/sbom/shippedscope/helpers_test.go | 10 + pkg/sbom/shippedscope/javascript_fileset.go | 80 ++++++ .../shippedscope/javascript_fileset_test.go | 100 ++++++++ pkg/sbom/shippedscope/lua_fileset.go | 241 ++++++++++++++++++ pkg/sbom/shippedscope/lua_fileset_test.go | 96 +++++++ pkg/sbom/shippedscope/os_pm_fileset.go | 69 +++++ pkg/sbom/shippedscope/os_pm_fileset_test.go | 74 ++++++ pkg/sbom/shippedscope/python_fileset.go | 122 +++++++++ pkg/sbom/shippedscope/python_fileset_test.go | 112 ++++++++ 10 files changed, 993 insertions(+) create mode 100644 pkg/sbom/shippedscope/fileset.go create mode 100644 pkg/sbom/shippedscope/javascript_fileset.go create mode 100644 pkg/sbom/shippedscope/javascript_fileset_test.go create mode 100644 pkg/sbom/shippedscope/lua_fileset.go create mode 100644 pkg/sbom/shippedscope/lua_fileset_test.go create mode 100644 pkg/sbom/shippedscope/os_pm_fileset.go create mode 100644 pkg/sbom/shippedscope/os_pm_fileset_test.go create mode 100644 pkg/sbom/shippedscope/python_fileset.go create mode 100644 pkg/sbom/shippedscope/python_fileset_test.go diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go new file mode 100644 index 0000000000..67c5ffccef --- /dev/null +++ b/pkg/sbom/shippedscope/fileset.go @@ -0,0 +1,89 @@ +package shippedscope + +import ( + "context" + "path" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type Ecosystem string + +const ( + EcosystemOSPM Ecosystem = "os-pm" + EcosystemPython Ecosystem = "python" + EcosystemJavaScript Ecosystem = "javascript" + EcosystemLua Ecosystem = "lua" +) + +// Fileset is one installed copy of a component and the files the image gives it. npm keeps +// several copies of one version under different paths, so Root tells them apart; the rule +// that consumes this treats them as one owner. +type Fileset struct { + Ecosystem Ecosystem + Name string + Version string + Root string + Files map[string]File + Symlinks map[string]string +} + +// CollectFilesets reads the image once and answers, for every component an ecosystem +// installed into it, which files are that component's. +// +// Each ecosystem draws the boundary itself, and none of them draw it around a directory: +// Python takes the paths its RECORD declares, JavaScript takes a package directory minus +// the nested package trees inside it, Lua takes the paths its rock manifest declares across +// the install roots they land in, and os-pm takes the list its package manager wrote. The +// digests always come from the image, never from the manifest that names the file. +func CollectFilesets(ctx context.Context, img v1.Image) ([]Fileset, error) { + index, manifests, err := indexImageFiles(ctx, img, isFilesetManifest) + if err != nil { + return nil, err + } + + filesets, err := osPMFilesets(ctx, index, manifests) + if err != nil { + return nil, err + } + + filesets = append(filesets, pythonFilesets(index, manifests)...) + filesets = append(filesets, javascriptFilesets(index, manifests)...) + filesets = append(filesets, luaFilesets(index, manifests)...) + + return filesets, nil +} + +func isFilesetManifest(file string) bool { + return file == pmFilesPath || + strings.HasSuffix(file, distInfoRecordSuffix) || + isNodePackageManifest(file) || + path.Base(file) == rockManifestName +} + +// collect gathers what the image holds at the given paths. A path a manifest declares but +// the image no longer holds is left out: the fileset describes what shipped, not what was +// installed. +func collect(index *FileIndex, paths []string) (map[string]File, map[string]string) { + files := make(map[string]File) + symlinks := make(map[string]string) + + for _, file := range paths { + if entry, ok := index.Files[file]; ok { + files[file] = entry + + continue + } + + if target, ok := index.Symlinks[file]; ok { + symlinks[file] = target + } + } + + return files, symlinks +} + +func rootedJoin(dir, rel string) string { + return rootedPath(path.Join(dir, rel)) +} diff --git a/pkg/sbom/shippedscope/helpers_test.go b/pkg/sbom/shippedscope/helpers_test.go index af8f4904ef..f187a49347 100644 --- a/pkg/sbom/shippedscope/helpers_test.go +++ b/pkg/sbom/shippedscope/helpers_test.go @@ -98,3 +98,13 @@ func digestOf(body string) string { return "sha256:" + hex.EncodeToString(sum[:]) } + +func filesetOf(filesets []Fileset, ecosystem Ecosystem, name, version string) *Fileset { + for i, fileset := range filesets { + if fileset.Ecosystem == ecosystem && fileset.Name == name && fileset.Version == version { + return &filesets[i] + } + } + + return nil +} diff --git a/pkg/sbom/shippedscope/javascript_fileset.go b/pkg/sbom/shippedscope/javascript_fileset.go new file mode 100644 index 0000000000..874e39a4dd --- /dev/null +++ b/pkg/sbom/shippedscope/javascript_fileset.go @@ -0,0 +1,80 @@ +package shippedscope + +import ( + "encoding/json" + "path" + "strings" +) + +const ( + nodeModulesDir = "/node_modules/" + distributionJSDir = "/usr/share/nodejs/" +) + +// javascriptFilesets draws a package's boundary around the directory holding its +// package.json, minus the package trees nested inside it: npm installs a second copy of a +// dependency under its dependent when versions conflict, and those files belong to that +// copy, not to the package above it. +func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { + var filesets []Fileset + + for manifest, body := range manifests { + if !isNodePackageManifest(manifest) { + continue + } + + var declaration struct { + Name string `json:"name"` + Version string `json:"version"` + } + if err := json.Unmarshal(body, &declaration); err != nil { + continue + } + if declaration.Name == "" || declaration.Version == "" { + continue + } + + root := path.Dir(manifest) + + files, symlinks := collect(index, packageTreePaths(index, root)) + + filesets = append(filesets, Fileset{ + Ecosystem: EcosystemJavaScript, + Name: declaration.Name, + Version: declaration.Version, + Root: root, + Files: files, + Symlinks: symlinks, + }) + } + + return filesets +} + +func isNodePackageManifest(file string) bool { + if path.Base(file) != "package.json" { + return false + } + + dir := path.Dir(file) + "/" + + return strings.Contains(dir, nodeModulesDir) || strings.Contains(dir, distributionJSDir) +} + +func packageTreePaths(index *FileIndex, root string) []string { + prefix := root + "/" + + var paths []string + for file := range index.Files { + if strings.HasPrefix(file, prefix) && !strings.Contains(strings.TrimPrefix(file, prefix), "node_modules/") { + paths = append(paths, file) + } + } + for file := range index.Symlinks { + if strings.HasPrefix(file, prefix) && !strings.Contains(strings.TrimPrefix(file, prefix), "node_modules/") { + paths = append(paths, file) + } + } + + return paths +} diff --git a/pkg/sbom/shippedscope/javascript_fileset_test.go b/pkg/sbom/shippedscope/javascript_fileset_test.go new file mode 100644 index 0000000000..0ee4261b28 --- /dev/null +++ b/pkg/sbom/shippedscope/javascript_fileset_test.go @@ -0,0 +1,100 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CollectFilesets, JavaScript", func() { + It("leaves a nested package out of the tree that holds it", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/express/package.json", `{"name":"express","version":"4.21.2"}`), + regularFile("app/node_modules/express/lib/router.js", "router"), + regularFile("app/node_modules/express/node_modules/debug/package.json", `{"name":"debug","version":"2.6.9"}`), + regularFile("app/node_modules/express/node_modules/debug/src/index.js", "debug"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + express := filesetOf(filesets, EcosystemJavaScript, "express", "4.21.2") + Expect(express).NotTo(BeNil()) + Expect(express.Files).To(HaveKey("/app/node_modules/express/lib/router.js")) + Expect(express.Files).NotTo(HaveKey("/app/node_modules/express/node_modules/debug/src/index.js")) + + debug := filesetOf(filesets, EcosystemJavaScript, "debug", "2.6.9") + Expect(debug).NotTo(BeNil()) + Expect(debug.Files).To(HaveKey("/app/node_modules/express/node_modules/debug/src/index.js")) + }) + + It("takes the name the package declares, not the directory it sits in", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/@babel/core/package.json", `{"name":"@babel/core","version":"7.28.0"}`), + regularFile("app/node_modules/@babel/core/lib/index.js", "babel"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + babel := filesetOf(filesets, EcosystemJavaScript, "@babel/core", "7.28.0") + Expect(babel).NotTo(BeNil()) + Expect(babel.Files).To(HaveKey("/app/node_modules/@babel/core/lib/index.js")) + }) + + It("takes packages a distribution installs outside node_modules", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/share/nodejs/semver/package.json", `{"name":"semver","version":"7.6.3"}`), + regularFile("usr/share/nodejs/semver/index.js", "semver"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + semver := filesetOf(filesets, EcosystemJavaScript, "semver", "7.6.3") + Expect(semver).NotTo(BeNil()) + Expect(semver.Files).To(HaveKey("/usr/share/nodejs/semver/index.js")) + }) + + It("keeps a package.json that belongs to no package tree out of the filesets", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/package.json", `{"name":"my-app","version":"1.0.0"}`), + regularFile("app/index.js", "app"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesetOf(filesets, EcosystemJavaScript, "my-app", "1.0.0")).To(BeNil()) + }) + + It("keeps the symlinks a package tree holds", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/semver/package.json", `{"name":"semver","version":"7.6.3"}`), + symlink("app/node_modules/semver/bin/semver", "../index.js"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + semver := filesetOf(filesets, EcosystemJavaScript, "semver", "7.6.3") + Expect(semver.Symlinks).To(HaveKeyWithValue("/app/node_modules/semver/bin/semver", "../index.js")) + }) + + It("ignores a package declaring neither name nor version", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/broken/package.json", `{"description":"no name here"}`), + regularFile("app/node_modules/broken/index.js", "broken"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets).To(BeEmpty()) + }) +}) diff --git a/pkg/sbom/shippedscope/lua_fileset.go b/pkg/sbom/shippedscope/lua_fileset.go new file mode 100644 index 0000000000..1bf7763069 --- /dev/null +++ b/pkg/sbom/shippedscope/lua_fileset.go @@ -0,0 +1,241 @@ +package shippedscope + +import ( + "path" + "strings" +) + +const rockManifestName = "rock_manifest" + +// luaFilesets draws a rock's boundary from its rock manifest. The rock's own directory +// holds little more than the rockspec and its docs: luarocks spreads the code that runs +// over trees shared with every other rock, and only the manifest says which files there +// are this rock's. +func luaFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { + var filesets []Fileset + + for manifest, body := range manifests { + if path.Base(manifest) != rockManifestName { + continue + } + + rockDir := path.Dir(manifest) + version := path.Base(rockDir) + name := path.Base(path.Dir(rockDir)) + + prefix, luaVersion, ok := luarocksTree(rockDir) + if !ok { + continue + } + + sections := parseRockManifest(body) + if len(sections) == 0 { + continue + } + + roots := map[string]string{ + "bin": path.Join(prefix, "bin"), + "lib": path.Join(prefix, "lib", "lua", luaVersion), + "lua": path.Join(prefix, "share", "lua", luaVersion), + } + + var declared []string + for section, entry := range sections { + root, ok := roots[section] + if !ok { + // Not an install root: the rock's own directory holds it, either as the + // file the section names or as a directory of them. + root = path.Join(rockDir, section) + } + + declared = append(declared, manifestEntryPaths(root, entry)...) + } + + files, symlinks := collect(index, declared) + + filesets = append(filesets, Fileset{ + Ecosystem: EcosystemLua, + Name: name, + Version: version, + Root: rockDir, + Files: files, + Symlinks: symlinks, + }) + } + + return filesets +} + +// manifestEntryPaths walks the nested tables a manifest uses for directories, down to the +// entries that name a file. +func manifestEntryPaths(prefix string, entry any) []string { + table, ok := entry.(map[string]any) + if !ok { + return []string{rootedPath(prefix)} + } + + var paths []string + for key, value := range table { + paths = append(paths, manifestEntryPaths(path.Join(prefix, key), value)...) + } + + return paths +} + +// luarocksTree splits a rock directory into the prefix the install roots hang off and the +// Lua version whose tree the rock was installed into: one image can carry several. +func luarocksTree(rockDir string) (string, string, bool) { + marker := "/lib/luarocks/rocks-" + + start := strings.LastIndex(rockDir, marker) + if start < 0 { + return "", "", false + } + + rest := rockDir[start+len(marker):] + luaVersion, _, ok := strings.Cut(rest, "/") + if !ok || luaVersion == "" { + return "", "", false + } + + return rockDir[:start], luaVersion, true +} + +// parseRockManifest reads the Lua table luarocks writes: nested tables of string keys and +// string values, and nothing else of the language. +func parseRockManifest(body []byte) map[string]any { + text := string(body) + + start := strings.Index(text, "{") + if start < 0 { + return nil + } + + table, _, ok := parseLuaTable(text[start:]) + if !ok { + return nil + } + + return table +} + +func parseLuaTable(text string) (map[string]any, int, bool) { + if len(text) == 0 || text[0] != '{' { + return nil, 0, false + } + + table := make(map[string]any) + + position := 1 + for position < len(text) { + switch { + case isLuaSpace(text[position]), text[position] == ',', text[position] == ';': + position++ + + continue + case text[position] == '}': + return table, position + 1, true + } + + key, consumed, ok := parseLuaKey(text[position:]) + if !ok { + return nil, 0, false + } + position += consumed + + for position < len(text) && isLuaSpace(text[position]) { + position++ + } + if position >= len(text) || text[position] != '=' { + return nil, 0, false + } + position++ + + for position < len(text) && isLuaSpace(text[position]) { + position++ + } + if position >= len(text) { + return nil, 0, false + } + + if text[position] == '{' { + nested, consumed, ok := parseLuaTable(text[position:]) + if !ok { + return nil, 0, false + } + + table[key] = nested + position += consumed + + continue + } + + value, consumed, ok := parseLuaString(text[position:]) + if !ok { + return nil, 0, false + } + + table[key] = value + position += consumed + } + + return nil, 0, false +} + +func parseLuaKey(text string) (string, int, bool) { + if text[0] == '[' { + value, consumed, ok := parseLuaString(text[1:]) + if !ok { + return "", 0, false + } + + position := 1 + consumed + for position < len(text) && isLuaSpace(text[position]) { + position++ + } + if position >= len(text) || text[position] != ']' { + return "", 0, false + } + + return value, position + 1, true + } + + end := 0 + for end < len(text) && (isLuaNameChar(text[end])) { + end++ + } + if end == 0 { + return "", 0, false + } + + return text[:end], end, true +} + +func parseLuaString(text string) (string, int, bool) { + if len(text) == 0 || (text[0] != '"' && text[0] != '\'') { + return "", 0, false + } + + quote := text[0] + for position := 1; position < len(text); position++ { + switch text[position] { + case '\\': + position++ + case quote: + return text[1:position], position + 1, true + } + } + + return "", 0, false +} + +func isLuaSpace(char byte) bool { + return char == ' ' || char == '\t' || char == '\n' || char == '\r' +} + +func isLuaNameChar(char byte) bool { + return char == '_' || + (char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') +} diff --git a/pkg/sbom/shippedscope/lua_fileset_test.go b/pkg/sbom/shippedscope/lua_fileset_test.go new file mode 100644 index 0000000000..b8ace87ae2 --- /dev/null +++ b/pkg/sbom/shippedscope/lua_fileset_test.go @@ -0,0 +1,96 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const cjsonRockManifest = `rock_manifest = { + bin = { + ["lua-cjson"] = "9f8c1a" + }, + doc = { + ["README.md"] = "2b7e4d" + }, + lib = { + ["cjson.so"] = "a1b2c3" + }, + lua = { + cjson = { + ["util.lua"] = "d4e5f6" + } + }, + ["lua-cjson-2.1.0-1.rockspec"] = "7a8b9c" +} +` + +var _ = Describe("CollectFilesets, Lua", func() { + It("spreads a rock over the install roots its manifest names", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/rock_manifest", cjsonRockManifest), + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/lua-cjson-2.1.0-1.rockspec", "spec"), + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/doc/README.md", "readme"), + regularFile("usr/local/bin/lua-cjson", "launcher"), + regularFile("usr/local/lib/lua/5.4/cjson.so", "native"), + regularFile("usr/local/share/lua/5.4/cjson/util.lua", "util"), + regularFile("usr/local/share/lua/5.4/other.lua", "not ours"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + cjson := filesetOf(filesets, EcosystemLua, "lua-cjson", "2.1.0-1") + Expect(cjson).NotTo(BeNil()) + Expect(cjson.Files).To(HaveKey("/usr/local/bin/lua-cjson")) + Expect(cjson.Files).To(HaveKey("/usr/local/lib/lua/5.4/cjson.so")) + Expect(cjson.Files).To(HaveKey("/usr/local/share/lua/5.4/cjson/util.lua")) + Expect(cjson.Files).NotTo(HaveKey("/usr/local/share/lua/5.4/other.lua")) + }) + + It("keeps the files the manifest names beside itself in the rock directory", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/rock_manifest", cjsonRockManifest), + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/lua-cjson-2.1.0-1.rockspec", "spec"), + regularFile("usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/doc/README.md", "readme"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + cjson := filesetOf(filesets, EcosystemLua, "lua-cjson", "2.1.0-1") + Expect(cjson.Files).To(HaveKey("/usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/lua-cjson-2.1.0-1.rockspec")) + Expect(cjson.Files).To(HaveKey("/usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/doc/README.md")) + }) + + It("takes the lua version of the tree the rock was installed into", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/luarocks/rocks-5.1/luafilesystem/1.8.0-1/rock_manifest", + "rock_manifest = {\n lib = {\n [\"lfs.so\"] = \"aaa\"\n }\n}\n"), + regularFile("usr/lib/lua/5.1/lfs.so", "native"), + regularFile("usr/lib/lua/5.4/lfs.so", "another tree"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + lfs := filesetOf(filesets, EcosystemLua, "luafilesystem", "1.8.0-1") + Expect(lfs.Files).To(HaveKey("/usr/lib/lua/5.1/lfs.so")) + Expect(lfs.Files).NotTo(HaveKey("/usr/lib/lua/5.4/lfs.so")) + }) + + It("ignores a manifest it cannot read", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/broken/1.0-1/rock_manifest", "this is not a lua table"), + regularFile("usr/local/share/lua/5.4/broken.lua", "code"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets).To(BeEmpty()) + }) +}) diff --git a/pkg/sbom/shippedscope/os_pm_fileset.go b/pkg/sbom/shippedscope/os_pm_fileset.go new file mode 100644 index 0000000000..2fd6e6c9e9 --- /dev/null +++ b/pkg/sbom/shippedscope/os_pm_fileset.go @@ -0,0 +1,69 @@ +package shippedscope + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/werf/logboek" +) + +const ( + pmIndexPath = "/var/lib/pm/index.json" + pmFilesPath = "/var/lib/pm/files.json" +) + +// pmFileList is what pm writes beside its index once it records the files it unpacks: +// one entry per installed package, its version, and the regular files and symlinks the +// package put into the image. +type pmFileList map[string]struct { + Version string `json:"version"` + Files map[string]string `json:"files"` + Symlinks map[string]string `json:"symlinks"` +} + +// osPMFilesets takes a package's boundary from the list pm wrote while unpacking it. The +// digest pm recorded is left aside: a file an install hook rewrote afterwards ships with +// the content the image holds now, which is what the index gives. +// +// An image installed by a pm too old to write the list yields no os-pm filesets, and a +// warning: breaking every build still on an older base would be the worse outcome. +func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][]byte) ([]Fileset, error) { + body, ok := manifests[pmFilesPath] + if !ok { + if _, indexed := index.Files[pmIndexPath]; indexed { + logboek.Context(ctx).Warn().LogF("WARNING: %s is present but %s is not: the image was installed by a pm that records no file lists, os-pm packages are left out of the filesets\n", pmIndexPath, pmFilesPath) + } + + return nil, nil + } + + var list pmFileList + if err := json.Unmarshal(body, &list); err != nil { + return nil, fmt.Errorf("parse %s: %w", pmFilesPath, err) + } + + var filesets []Fileset + for name, entry := range list { + declared := make([]string, 0, len(entry.Files)+len(entry.Symlinks)) + for file := range entry.Files { + declared = append(declared, rootedPath(file)) + } + for link := range entry.Symlinks { + declared = append(declared, rootedPath(link)) + } + + files, symlinks := collect(index, declared) + + filesets = append(filesets, Fileset{ + Ecosystem: EcosystemOSPM, + Name: name, + Version: entry.Version, + Root: "/", + Files: files, + Symlinks: symlinks, + }) + } + + return filesets, nil +} diff --git a/pkg/sbom/shippedscope/os_pm_fileset_test.go b/pkg/sbom/shippedscope/os_pm_fileset_test.go new file mode 100644 index 0000000000..5fa43d486c --- /dev/null +++ b/pkg/sbom/shippedscope/os_pm_fileset_test.go @@ -0,0 +1,74 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CollectFilesets, os-pm", func() { + It("takes the files and symlinks the package manager recorded", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/git", "git"), + regularFile("usr/lib64/libz.so.1.3.1", "zlib"), + symlink("usr/lib64/libz.so.1", "libz.so.1.3.1"), + regularFile("var/lib/pm/files.json", `{ + "git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead"}}, + "zlib": { + "version": "1.3.1", + "files": {"/usr/lib64/libz.so.1.3.1": "sha256:beef"}, + "symlinks": {"/usr/lib64/libz.so.1": "libz.so.1.3.1"} + } + }`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + git := filesetOf(filesets, EcosystemOSPM, "git", "2.50.1") + Expect(git).NotTo(BeNil()) + Expect(git.Files).To(HaveKey("/usr/bin/git")) + + zlib := filesetOf(filesets, EcosystemOSPM, "zlib", "1.3.1") + Expect(zlib.Files).To(HaveKey("/usr/lib64/libz.so.1.3.1")) + Expect(zlib.Symlinks).To(HaveKeyWithValue("/usr/lib64/libz.so.1", "libz.so.1.3.1")) + }) + + It("takes the digest the image holds, not the one the package manager recorded", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("etc/ssl/certs/ca-certificates.crt", "rebuilt after install"), + regularFile("var/lib/pm/files.json", + `{"ca-certificates": {"version": "20250419", "files": {"/etc/ssl/certs/ca-certificates.crt": "sha256:stale"}}}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + certs := filesetOf(filesets, EcosystemOSPM, "ca-certificates", "20250419") + Expect(certs.Files["/etc/ssl/certs/ca-certificates.crt"].Digest).To(Equal(digestOf("rebuilt after install"))) + }) + + It("carries on without the file list an older package manager never wrote", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/git", "git"), + regularFile("var/lib/pm/index.json", `{"git": {"name": "git", "version": "2.50.1"}}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets).To(BeEmpty()) + }) + + It("reports an image whose file list it cannot read", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("var/lib/pm/files.json", "{ this is not json"), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = CollectFilesets(ctx, img) + Expect(err).To(MatchError(ContainSubstring("files.json"))) + }) +}) diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go new file mode 100644 index 0000000000..6ee63730d4 --- /dev/null +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -0,0 +1,122 @@ +package shippedscope + +import ( + "encoding/csv" + "io" + "path" + "strings" +) + +const distInfoRecordSuffix = ".dist-info/RECORD" + +// pythonFilesets draws every distribution's boundary from the paths its RECORD declares. +// A directory would be the wrong boundary: PEP 420 namespace packages let several +// distributions write into one directory, and a distribution's code rarely sits in a +// directory named after it. +// +// Names are normalised the PEP 503 way, so a fileset of "PyYAML" is named "pyyaml". +func pythonFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { + var filesets []Fileset + + pycache := compiledSources(index) + + for manifest, body := range manifests { + if !strings.HasSuffix(manifest, distInfoRecordSuffix) { + continue + } + + distInfo := path.Dir(manifest) + name, version, ok := strings.Cut(strings.TrimSuffix(path.Base(distInfo), ".dist-info"), "-") + if !ok { + continue + } + + siteDir := path.Dir(distInfo) + + var declared []string + for _, record := range recordPaths(body) { + file := rootedJoin(siteDir, record) + declared = append(declared, file) + declared = append(declared, pycache[file]...) + } + + files, symlinks := collect(index, declared) + + filesets = append(filesets, Fileset{ + Ecosystem: EcosystemPython, + Name: normalizePythonName(name), + Version: version, + Root: distInfo, + Files: files, + Symlinks: symlinks, + }) + } + + return filesets +} + +func recordPaths(body []byte) []string { + reader := csv.NewReader(strings.NewReader(string(body))) + reader.FieldsPerRecord = -1 + reader.LazyQuotes = true + + var paths []string + for { + record, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + + continue + } + + if len(record) == 0 || record[0] == "" { + continue + } + + paths = append(paths, record[0]) + } + + return paths +} + +// compiledSources maps a source file to the bytecode compiled from it. A RECORD never +// declares bytecode an installer did not write, and an image that ran compileall — or that +// dropped the sources afterwards — holds it all the same. +func compiledSources(index *FileIndex) map[string][]string { + compiled := make(map[string][]string) + + for file := range index.Files { + if !strings.HasSuffix(file, ".pyc") { + continue + } + + dir := path.Dir(file) + if path.Base(dir) != "__pycache__" { + continue + } + + stem, _, _ := strings.Cut(path.Base(file), ".") + source := path.Join(path.Dir(dir), stem+".py") + compiled[source] = append(compiled[source], file) + } + + return compiled +} + +func normalizePythonName(name string) string { + normalized := strings.Map(func(r rune) rune { + if r == '_' || r == '.' { + return '-' + } + + return r + }, strings.ToLower(name)) + + for strings.Contains(normalized, "--") { + normalized = strings.ReplaceAll(normalized, "--", "-") + } + + return normalized +} diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go new file mode 100644 index 0000000000..a4ac0d09e6 --- /dev/null +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -0,0 +1,112 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CollectFilesets, Python", func() { + It("splits a namespace directory between the distributions that declare its files", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/google/auth/__init__.py", "auth"), + regularFile("usr/lib/python3.12/site-packages/google/protobuf/__init__.py", "proto"), + regularFile("usr/lib/python3.12/site-packages/google_auth-2.41.0.dist-info/RECORD", + "google/auth/__init__.py,sha256=aaa,4\n"), + regularFile("usr/lib/python3.12/site-packages/protobuf-6.33.0.dist-info/RECORD", + "google/protobuf/__init__.py,sha256=bbb,5\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + auth := filesetOf(filesets, EcosystemPython, "google-auth", "2.41.0") + Expect(auth).NotTo(BeNil()) + Expect(auth.Files).To(HaveKey("/usr/lib/python3.12/site-packages/google/auth/__init__.py")) + Expect(auth.Files).NotTo(HaveKey("/usr/lib/python3.12/site-packages/google/protobuf/__init__.py")) + + protobuf := filesetOf(filesets, EcosystemPython, "protobuf", "6.33.0") + Expect(protobuf).NotTo(BeNil()) + Expect(protobuf.Files).To(HaveLen(1)) + }) + + It("names the distribution the way PEP 503 spells it, not the way its metadata does", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/typing_extensions.py", "typing"), + regularFile("usr/lib/python3.12/site-packages/Typing_Extensions-4.12.2.dist-info/RECORD", + "typing_extensions.py,sha256=aaa,6\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + typing := filesetOf(filesets, EcosystemPython, "typing-extensions", "4.12.2") + Expect(typing).NotTo(BeNil()) + Expect(typing.Files).To(HaveKey("/usr/lib/python3.12/site-packages/typing_extensions.py")) + }) + + It("credits a compiled file to the source its distribution declares", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/yaml/__init__.py", "yaml"), + regularFile("usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc", "compiled"), + regularFile("usr/lib/python3.12/site-packages/PyYAML-6.0.2.dist-info/RECORD", + "yaml/__init__.py,sha256=aaa,4\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml.Files).To(HaveKey("/usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc")) + }) + + It("keeps the compiled file when the source it was built from is gone", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc", "compiled"), + regularFile("usr/lib/python3.12/site-packages/PyYAML-6.0.2.dist-info/RECORD", + "yaml/__init__.py,sha256=aaa,4\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml.Files).To(HaveLen(1)) + Expect(pyyaml.Files).To(HaveKey("/usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc")) + }) + + It("follows a record out of the site directory and keeps a declared symlink", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/pytest", "entrypoint"), + symlink("usr/lib/python3.12/site-packages/pytest/shortcut", "/usr/bin/pytest"), + regularFile("usr/lib/python3.12/site-packages/pytest-8.4.2.dist-info/RECORD", + "../../../bin/pytest,sha256=aaa,10\npytest/shortcut,,\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + pytest := filesetOf(filesets, EcosystemPython, "pytest", "8.4.2") + Expect(pytest.Files).To(HaveKey("/usr/bin/pytest")) + Expect(pytest.Symlinks).To(HaveKeyWithValue("/usr/lib/python3.12/site-packages/pytest/shortcut", "/usr/bin/pytest")) + }) + + It("leaves out records no longer in the image", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/yaml/__init__.py", "yaml"), + regularFile("usr/lib/python3.12/site-packages/PyYAML-6.0.2.dist-info/RECORD", + "yaml/__init__.py,sha256=aaa,4\nyaml/removed.py,sha256=bbb,9\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml.Files).To(HaveLen(1)) + }) +}) From f66f377c5d7e8353f326fe6baa8c1f947cd85caa Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Wed, 14 Oct 2026 16:45:18 +0300 Subject: [PATCH 3/5] fix(sbom): draw the npm boundary where an installer puts packages, and read hardlinked manifests Review of #435. A package.json anywhere under node_modules counted as a package, so the fixtures and examples packages ship became phantom components that also claimed the package's own files; only a direct child of node_modules, a scope directory in it, or a distribution's nodejs directory counts now. A nested-tree check matched a directory merely ending in node_modules. A manifest the image reaches through a hardlink got its digest but not its bytes, so a hardlinked files.json read as "pm too old"; captured bytes now travel with the link, across layers when needed. CollectFilesets returns whether os-pm packages lack a file list instead of only logging it, so a consumer can tell that from an image without os-pm packages. Lua manifest strings are unescaped the way luarocks escapes them. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/file_index.go | 126 +++++++++++++----- pkg/sbom/shippedscope/fileset.go | 34 +++-- pkg/sbom/shippedscope/helpers_test.go | 6 +- pkg/sbom/shippedscope/javascript_fileset.go | 54 ++++++-- .../shippedscope/javascript_fileset_test.go | 42 +++++- pkg/sbom/shippedscope/lua_fileset.go | 18 ++- pkg/sbom/shippedscope/lua_fileset_test.go | 20 ++- pkg/sbom/shippedscope/os_pm_fileset.go | 20 +-- pkg/sbom/shippedscope/os_pm_fileset_test.go | 72 +++++++++- pkg/sbom/shippedscope/python_fileset.go | 9 +- pkg/sbom/shippedscope/python_fileset_test.go | 33 ++++- 11 files changed, 358 insertions(+), 76 deletions(-) diff --git a/pkg/sbom/shippedscope/file_index.go b/pkg/sbom/shippedscope/file_index.go index 6892b3945a..e4be119b3f 100644 --- a/pkg/sbom/shippedscope/file_index.go +++ b/pkg/sbom/shippedscope/file_index.go @@ -30,6 +30,11 @@ type FileIndex struct { Files map[string]File // Symlinks holds targets exactly as the image records them: possibly relative, possibly dangling. Symlinks map[string]string + + // bodies holds the captured content by indexed name; targetBodies holds, by target + // path, the content a hardlink's name asked for while the layers were read. + bodies map[string][]byte + targetBodies map[string][]byte } // IndexImageFiles records what the image ships: every regular file by rooted path, with @@ -51,14 +56,20 @@ func IndexImageFiles(ctx context.Context, img v1.Image) (*FileIndex, error) { } // capture names the files whose bytes the caller needs beside their digest; they are read -// once, while the file passes by, instead of costing another walk of the image. +// once, while the file passes by, instead of costing another walk of the image. The bytes +// travel with the entry, so a hardlink to such a file gets them along with its digest. A +// symlink to one does not: it stays a symlink in the index, as everywhere else. func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string) bool) (*FileIndex, map[string][]byte, error) { index := &FileIndex{ - Files: make(map[string]File), - Symlinks: make(map[string]string), + Files: make(map[string]File), + Symlinks: make(map[string]string), + bodies: make(map[string][]byte), + targetBodies: make(map[string][]byte), } hardlinks := make(map[string]string) - captured := make(map[string][]byte) + if capture == nil { + capture = func(string) bool { return false } + } rootfs := mutate.Extract(img) defer rootfs.Close() @@ -84,24 +95,14 @@ func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string switch header.Typeflag { case tar.TypeReg: - hasher := sha256.New() - var body bytes.Buffer - - var sink io.Writer = hasher - if capture != nil && capture(name) { - sink = io.MultiWriter(hasher, &body) - } - - if _, err := io.Copy(sink, reader); err != nil { + file, body, err := readFile(reader, header, capture(name)) + if err != nil { return nil, nil, fmt.Errorf("hash %q: %w", name, err) } - index.Files[name] = File{ - Digest: "sha256:" + hex.EncodeToString(hasher.Sum(nil)), - Size: header.Size, - } - if sink != io.Writer(hasher) { - captured[name] = body.Bytes() + index.Files[name] = file + if body != nil { + index.bodies[name] = body } case tar.TypeSymlink: index.Symlinks[name] = header.Linkname @@ -118,16 +119,44 @@ func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string return nil, nil, fmt.Errorf("read image filesystem: %w", err) } - remaining, err := resolveHardlinksWithinLayers(ctx, img, index, hardlinks) + remaining, err := resolveHardlinksWithinLayers(ctx, img, index, hardlinks, capture) if err != nil { return nil, nil, err } resolveHardlinks(ctx, index, remaining) + captured := index.bodies + index.bodies = nil + index.targetBodies = nil + return index, captured, nil } +func readFile(reader io.Reader, header *tar.Header, keepBody bool) (File, []byte, error) { + hasher := sha256.New() + + var body bytes.Buffer + var sink io.Writer = hasher + if keepBody { + sink = io.MultiWriter(hasher, &body) + } + + if _, err := io.Copy(sink, reader); err != nil { + return File{}, nil, err + } + + file := File{ + Digest: "sha256:" + hex.EncodeToString(hasher.Sum(nil)), + Size: header.Size, + } + if !keepBody { + return file, nil, nil + } + + return file, body.Bytes(), nil +} + // A layer tar spells a hardlink as a regular file plus a link entry beside it, so the bytes // it names are the ones its own layer holds. Extract flattens that boundary away, and a // lookup in the flattened index would credit the link with whatever an upper layer later @@ -135,16 +164,18 @@ func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string // sixth of the first pass on uncompressed layers and at nearly its equal on compressed ones, // where the gzip dominates — so it runs only for images that hold hardlinks and stops as // soon as every link has been answered. -func resolveHardlinksWithinLayers(ctx context.Context, img v1.Image, index *FileIndex, hardlinks map[string]string) (map[string]string, error) { +func resolveHardlinksWithinLayers(ctx context.Context, img v1.Image, index *FileIndex, hardlinks map[string]string, capture func(path string) bool) (map[string]string, error) { if len(hardlinks) == 0 { return hardlinks, nil } + // A target is read for the link's sake, so it is the link's name that decides whether + // the bytes are kept. targets := make(map[string]bool, len(hardlinks)) remaining := make(map[string]string, len(hardlinks)) for name, target := range hardlinks { remaining[name] = target - targets[target] = true + targets[target] = targets[target] || capture(name) } layers, err := img.Layers() @@ -153,7 +184,11 @@ func resolveHardlinksWithinLayers(ctx context.Context, img v1.Image, index *File } settled := make(map[string]bool, len(hardlinks)) - for i := len(layers) - 1; i >= 0 && len(settled) < len(hardlinks); i-- { + for i := len(layers) - 1; i >= 0; i-- { + if len(settled) == len(hardlinks) && !bodyOwed(index, remaining, capture) { + break + } + if err := resolveHardlinksInLayer(ctx, layers[i], index, remaining, targets, settled); err != nil { return nil, err } @@ -162,6 +197,22 @@ func resolveHardlinksWithinLayers(ctx context.Context, img v1.Image, index *File return remaining, nil } +// A link its own layer could not answer is resolved against the flattened view later, and +// the bytes its name asked for have to come from the layer that holds the target. +func bodyOwed(index *FileIndex, remaining map[string]string, capture func(path string) bool) bool { + for name, target := range remaining { + if !capture(name) { + continue + } + + if _, ok := index.targetBodies[target]; !ok { + return true + } + } + + return false +} + func resolveHardlinksInLayer(ctx context.Context, layer v1.Layer, index *FileIndex, remaining map[string]string, targets, settled map[string]bool) error { contents, err := layer.Uncompressed() if err != nil { @@ -170,6 +221,7 @@ func resolveHardlinksInLayer(ctx context.Context, layer v1.Layer, index *FileInd defer contents.Close() files := make(map[string]File) + bodies := make(map[string][]byte) symlinks := make(map[string]string) owned := make(map[string]string) @@ -181,7 +233,7 @@ func resolveHardlinksInLayer(ctx context.Context, layer v1.Layer, index *FileInd header, err := reader.Next() if errors.Is(err, io.EOF) { - resolveOwnedHardlinks(index, remaining, owned, files, symlinks) + resolveOwnedHardlinks(index, remaining, owned, files, bodies, symlinks) return nil } @@ -193,21 +245,25 @@ func resolveHardlinksInLayer(ctx context.Context, layer v1.Layer, index *FileInd switch header.Typeflag { case tar.TypeReg: - if !targets[name] { + keepBody, ok := targets[name] + if !ok { continue } - hasher := sha256.New() - if _, err := io.Copy(hasher, reader); err != nil { + file, body, err := readFile(reader, header, keepBody) + if err != nil { return fmt.Errorf("hash %q: %w", name, err) } - files[name] = File{ - Digest: "sha256:" + hex.EncodeToString(hasher.Sum(nil)), - Size: header.Size, + files[name] = file + if body != nil { + bodies[name] = body + if _, ok := index.targetBodies[name]; !ok { + index.targetBodies[name] = body + } } case tar.TypeSymlink: - if targets[name] { + if _, ok := targets[name]; ok { symlinks[name] = header.Linkname } case tar.TypeLink: @@ -226,10 +282,13 @@ func resolveHardlinksInLayer(ctx context.Context, layer v1.Layer, index *FileInd // A link entry may precede the target it names, so the layer answers only once its tar has // been read to the end. -func resolveOwnedHardlinks(index *FileIndex, remaining, owned map[string]string, files map[string]File, symlinks map[string]string) { +func resolveOwnedHardlinks(index *FileIndex, remaining, owned map[string]string, files map[string]File, bodies map[string][]byte, symlinks map[string]string) { for name, target := range owned { if file, ok := files[target]; ok { index.Files[name] = file + if body, ok := bodies[target]; ok { + index.bodies[name] = body + } delete(remaining, name) continue @@ -270,6 +329,9 @@ func resolveHardlinks(ctx context.Context, index *FileIndex, hardlinks map[strin for { if file, ok := index.Files[target]; ok { index.Files[name] = file + if body, ok := index.targetBodies[target]; ok { + index.bodies[name] = body + } break } if link, ok := index.Symlinks[target]; ok { diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go index 67c5ffccef..cf7101baa5 100644 --- a/pkg/sbom/shippedscope/fileset.go +++ b/pkg/sbom/shippedscope/fileset.go @@ -18,17 +18,24 @@ const ( ) // Fileset is one installed copy of a component and the files the image gives it. npm keeps -// several copies of one version under different paths, so Root tells them apart; the rule -// that consumes this treats them as one owner. +// several copies of one version under different paths, so one (Ecosystem, Name, Version) +// can appear more than once; the rule that consumes this treats them as one owner. type Fileset struct { Ecosystem Ecosystem Name string Version string - Root string Files map[string]File Symlinks map[string]string } +type Filesets struct { + Filesets []Fileset + // OSPMFileListMissing is set when the image carries os-pm packages but was installed + // by a pm that recorded no file list for them, so none of them has a fileset here. A + // consumer that would drop a component without a fileset must not do so for os-pm then. + OSPMFileListMissing bool +} + // CollectFilesets reads the image once and answers, for every component an ecosystem // installed into it, which files are that component's. // @@ -37,20 +44,27 @@ type Fileset struct { // the nested package trees inside it, Lua takes the paths its rock manifest declares across // the install roots they land in, and os-pm takes the list its package manager wrote. The // digests always come from the image, never from the manifest that names the file. -func CollectFilesets(ctx context.Context, img v1.Image) ([]Fileset, error) { +// +// A component installed some other way — a Python egg-info, a LuaRocks 2 tree, a Yarn PnP +// cache — has no fileset, and this says nothing about whether it shipped. +func CollectFilesets(ctx context.Context, img v1.Image) (*Filesets, error) { index, manifests, err := indexImageFiles(ctx, img, isFilesetManifest) if err != nil { return nil, err } - filesets, err := osPMFilesets(ctx, index, manifests) + osPM, missing, err := osPMFilesets(ctx, index, manifests) if err != nil { return nil, err } - filesets = append(filesets, pythonFilesets(index, manifests)...) - filesets = append(filesets, javascriptFilesets(index, manifests)...) - filesets = append(filesets, luaFilesets(index, manifests)...) + filesets := &Filesets{ + Filesets: osPM, + OSPMFileListMissing: missing, + } + filesets.Filesets = append(filesets.Filesets, pythonFilesets(index, manifests)...) + filesets.Filesets = append(filesets.Filesets, javascriptFilesets(index, manifests)...) + filesets.Filesets = append(filesets.Filesets, luaFilesets(index, manifests)...) return filesets, nil } @@ -87,3 +101,7 @@ func collect(index *FileIndex, paths []string) (map[string]File, map[string]stri func rootedJoin(dir, rel string) string { return rootedPath(path.Join(dir, rel)) } + +func hasPathPrefix(file, dir string) bool { + return strings.HasPrefix(file, dir+"/") +} diff --git a/pkg/sbom/shippedscope/helpers_test.go b/pkg/sbom/shippedscope/helpers_test.go index f187a49347..f3f52369f6 100644 --- a/pkg/sbom/shippedscope/helpers_test.go +++ b/pkg/sbom/shippedscope/helpers_test.go @@ -99,10 +99,10 @@ func digestOf(body string) string { return "sha256:" + hex.EncodeToString(sum[:]) } -func filesetOf(filesets []Fileset, ecosystem Ecosystem, name, version string) *Fileset { - for i, fileset := range filesets { +func filesetOf(filesets *Filesets, ecosystem Ecosystem, name, version string) *Fileset { + for i, fileset := range filesets.Filesets { if fileset.Ecosystem == ecosystem && fileset.Name == name && fileset.Version == version { - return &filesets[i] + return &filesets.Filesets[i] } } diff --git a/pkg/sbom/shippedscope/javascript_fileset.go b/pkg/sbom/shippedscope/javascript_fileset.go index 874e39a4dd..204c6b826d 100644 --- a/pkg/sbom/shippedscope/javascript_fileset.go +++ b/pkg/sbom/shippedscope/javascript_fileset.go @@ -6,15 +6,20 @@ import ( "strings" ) -const ( - nodeModulesDir = "/node_modules/" - distributionJSDir = "/usr/share/nodejs/" -) +const nodeModulesDir = "node_modules" + +// Where a distribution's packages for Node.js land when installed by the system package +// manager rather than npm. +var distributionNodeDirs = []string{"/usr/share/nodejs", "/usr/lib/nodejs"} // javascriptFilesets draws a package's boundary around the directory holding its // package.json, minus the package trees nested inside it: npm installs a second copy of a // dependency under its dependent when versions conflict, and those files belong to that // copy, not to the package above it. +// +// Only a package.json sitting where an installer puts packages counts. Packages ship +// fixtures and examples with a package.json of their own, and those stay the package's +// files. A Yarn PnP cache holds packages as archives and is not read. func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { var filesets []Fileset @@ -42,7 +47,6 @@ func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset Ecosystem: EcosystemJavaScript, Name: declaration.Name, Version: declaration.Version, - Root: root, Files: files, Symlinks: symlinks, }) @@ -51,30 +55,58 @@ func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset return filesets } +// isNodePackageManifest accepts //package.json and +// /@//package.json, where is any node_modules +// directory or one of the distribution directories. func isNodePackageManifest(file string) bool { if path.Base(file) != "package.json" { return false } - dir := path.Dir(file) + "/" + packages := path.Dir(path.Dir(file)) + if strings.HasPrefix(path.Base(packages), "@") { + packages = path.Dir(packages) + } + + if path.Base(packages) == nodeModulesDir { + return true + } - return strings.Contains(dir, nodeModulesDir) || strings.Contains(dir, distributionJSDir) + for _, dir := range distributionNodeDirs { + if packages == dir { + return true + } + } + + return false } func packageTreePaths(index *FileIndex, root string) []string { - prefix := root + "/" - var paths []string for file := range index.Files { - if strings.HasPrefix(file, prefix) && !strings.Contains(strings.TrimPrefix(file, prefix), "node_modules/") { + if inPackageTree(file, root) { paths = append(paths, file) } } for file := range index.Symlinks { - if strings.HasPrefix(file, prefix) && !strings.Contains(strings.TrimPrefix(file, prefix), "node_modules/") { + if inPackageTree(file, root) { paths = append(paths, file) } } return paths } + +func inPackageTree(file, root string) bool { + if !hasPathPrefix(file, root) { + return false + } + + for _, segment := range strings.Split(path.Dir(strings.TrimPrefix(file, root+"/")), "/") { + if segment == nodeModulesDir { + return false + } + } + + return true +} diff --git a/pkg/sbom/shippedscope/javascript_fileset_test.go b/pkg/sbom/shippedscope/javascript_fileset_test.go index 0ee4261b28..bcf2ec4e0a 100644 --- a/pkg/sbom/shippedscope/javascript_fileset_test.go +++ b/pkg/sbom/shippedscope/javascript_fileset_test.go @@ -47,6 +47,8 @@ var _ = Describe("CollectFilesets, JavaScript", func() { img, err := imageOf([]tarEntry{ regularFile("usr/share/nodejs/semver/package.json", `{"name":"semver","version":"7.6.3"}`), regularFile("usr/share/nodejs/semver/index.js", "semver"), + regularFile("usr/lib/nodejs/@types/node/package.json", `{"name":"@types/node","version":"22.0.0"}`), + regularFile("usr/lib/nodejs/@types/node/index.d.ts", "types"), }) Expect(err).NotTo(HaveOccurred()) @@ -56,6 +58,43 @@ var _ = Describe("CollectFilesets, JavaScript", func() { semver := filesetOf(filesets, EcosystemJavaScript, "semver", "7.6.3") Expect(semver).NotTo(BeNil()) Expect(semver.Files).To(HaveKey("/usr/share/nodejs/semver/index.js")) + + types := filesetOf(filesets, EcosystemJavaScript, "@types/node", "22.0.0") + Expect(types).NotTo(BeNil()) + Expect(types.Files).To(HaveKey("/usr/lib/nodejs/@types/node/index.d.ts")) + }) + + It("leaves a fixture a package ships inside its own tree to that package", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/resolve/package.json", `{"name":"resolve","version":"1.22.8"}`), + regularFile("app/node_modules/resolve/test/resolver/baz/package.json", `{"name":"baz","version":"1.0.0"}`), + regularFile("app/node_modules/resolve/test/resolver/baz/quux.js", "fixture"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesetOf(filesets, EcosystemJavaScript, "baz", "1.0.0")).To(BeNil()) + + resolve := filesetOf(filesets, EcosystemJavaScript, "resolve", "1.22.8") + Expect(resolve).NotTo(BeNil()) + Expect(resolve.Files).To(HaveKey("/app/node_modules/resolve/test/resolver/baz/quux.js")) + }) + + It("tells a nested package tree from a directory whose name merely ends the same way", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("app/node_modules/webpack/package.json", `{"name":"webpack","version":"5.95.0"}`), + regularFile("app/node_modules/webpack/lib/my_node_modules/plugin.js", "plugin"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + webpack := filesetOf(filesets, EcosystemJavaScript, "webpack", "5.95.0") + Expect(webpack).NotTo(BeNil()) + Expect(webpack.Files).To(HaveKey("/app/node_modules/webpack/lib/my_node_modules/plugin.js")) }) It("keeps a package.json that belongs to no package tree out of the filesets", func(ctx SpecContext) { @@ -82,6 +121,7 @@ var _ = Describe("CollectFilesets, JavaScript", func() { Expect(err).NotTo(HaveOccurred()) semver := filesetOf(filesets, EcosystemJavaScript, "semver", "7.6.3") + Expect(semver).NotTo(BeNil()) Expect(semver.Symlinks).To(HaveKeyWithValue("/app/node_modules/semver/bin/semver", "../index.js")) }) @@ -95,6 +135,6 @@ var _ = Describe("CollectFilesets, JavaScript", func() { filesets, err := CollectFilesets(ctx, img) Expect(err).NotTo(HaveOccurred()) - Expect(filesets).To(BeEmpty()) + Expect(filesets.Filesets).To(BeEmpty()) }) }) diff --git a/pkg/sbom/shippedscope/lua_fileset.go b/pkg/sbom/shippedscope/lua_fileset.go index 1bf7763069..b0513ed443 100644 --- a/pkg/sbom/shippedscope/lua_fileset.go +++ b/pkg/sbom/shippedscope/lua_fileset.go @@ -57,7 +57,6 @@ func luaFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { Ecosystem: EcosystemLua, Name: name, Version: version, - Root: rockDir, Files: files, Symlinks: symlinks, }) @@ -83,7 +82,9 @@ func manifestEntryPaths(prefix string, entry any) []string { } // luarocksTree splits a rock directory into the prefix the install roots hang off and the -// Lua version whose tree the rock was installed into: one image can carry several. +// Lua version whose tree the rock was installed into: one image can carry several. The +// unversioned tree LuaRocks 2 wrote keeps the Lua version in its config instead, and is +// not read. func luarocksTree(rockDir string) (string, string, bool) { marker := "/lib/luarocks/rocks-" @@ -211,18 +212,29 @@ func parseLuaKey(text string) (string, int, bool) { return text[:end], end, true } +// parseLuaString reads a string the way luarocks writes one: %q-quoted, so a quote, a +// backslash and a newline inside it come escaped. func parseLuaString(text string) (string, int, bool) { if len(text) == 0 || (text[0] != '"' && text[0] != '\'') { return "", 0, false } quote := text[0] + + var value strings.Builder for position := 1; position < len(text); position++ { switch text[position] { case '\\': position++ + if position >= len(text) { + return "", 0, false + } + + value.WriteByte(text[position]) case quote: - return text[1:position], position + 1, true + return value.String(), position + 1, true + default: + value.WriteByte(text[position]) } } diff --git a/pkg/sbom/shippedscope/lua_fileset_test.go b/pkg/sbom/shippedscope/lua_fileset_test.go index b8ace87ae2..7a499cebef 100644 --- a/pkg/sbom/shippedscope/lua_fileset_test.go +++ b/pkg/sbom/shippedscope/lua_fileset_test.go @@ -60,6 +60,7 @@ var _ = Describe("CollectFilesets, Lua", func() { Expect(err).NotTo(HaveOccurred()) cjson := filesetOf(filesets, EcosystemLua, "lua-cjson", "2.1.0-1") + Expect(cjson).NotTo(BeNil()) Expect(cjson.Files).To(HaveKey("/usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/lua-cjson-2.1.0-1.rockspec")) Expect(cjson.Files).To(HaveKey("/usr/local/lib/luarocks/rocks-5.4/lua-cjson/2.1.0-1/doc/README.md")) }) @@ -77,10 +78,27 @@ var _ = Describe("CollectFilesets, Lua", func() { Expect(err).NotTo(HaveOccurred()) lfs := filesetOf(filesets, EcosystemLua, "luafilesystem", "1.8.0-1") + Expect(lfs).NotTo(BeNil()) Expect(lfs.Files).To(HaveKey("/usr/lib/lua/5.1/lfs.so")) Expect(lfs.Files).NotTo(HaveKey("/usr/lib/lua/5.4/lfs.so")) }) + It("reads a name the manifest had to escape", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/odd/1.0-1/rock_manifest", + "rock_manifest = {\n lua = {\n [\"say \\\"hi\\\".lua\"] = \"aaa\"\n }\n}\n"), + regularFile("usr/local/share/lua/5.4/say \"hi\".lua", "code"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + odd := filesetOf(filesets, EcosystemLua, "odd", "1.0-1") + Expect(odd).NotTo(BeNil()) + Expect(odd.Files).To(HaveKey("/usr/local/share/lua/5.4/say \"hi\".lua")) + }) + It("ignores a manifest it cannot read", func(ctx SpecContext) { img, err := imageOf([]tarEntry{ regularFile("usr/local/lib/luarocks/rocks-5.4/broken/1.0-1/rock_manifest", "this is not a lua table"), @@ -91,6 +109,6 @@ var _ = Describe("CollectFilesets, Lua", func() { filesets, err := CollectFilesets(ctx, img) Expect(err).NotTo(HaveOccurred()) - Expect(filesets).To(BeEmpty()) + Expect(filesets.Filesets).To(BeEmpty()) }) }) diff --git a/pkg/sbom/shippedscope/os_pm_fileset.go b/pkg/sbom/shippedscope/os_pm_fileset.go index 2fd6e6c9e9..add4c74cba 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset.go +++ b/pkg/sbom/shippedscope/os_pm_fileset.go @@ -26,21 +26,24 @@ type pmFileList map[string]struct { // digest pm recorded is left aside: a file an install hook rewrote afterwards ships with // the content the image holds now, which is what the index gives. // -// An image installed by a pm too old to write the list yields no os-pm filesets, and a -// warning: breaking every build still on an older base would be the worse outcome. -func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][]byte) ([]Fileset, error) { +// An image installed by a pm too old to write the list yields no os-pm filesets, a +// warning, and the second result set: breaking every build still on an older base would +// be the worse outcome. +func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][]byte) ([]Fileset, bool, error) { body, ok := manifests[pmFilesPath] if !ok { - if _, indexed := index.Files[pmIndexPath]; indexed { - logboek.Context(ctx).Warn().LogF("WARNING: %s is present but %s is not: the image was installed by a pm that records no file lists, os-pm packages are left out of the filesets\n", pmIndexPath, pmFilesPath) + if _, indexed := index.Files[pmIndexPath]; !indexed { + return nil, false, nil } - return nil, nil + logboek.Context(ctx).Warn().LogF("WARNING: %s is present but %s is not: the image was installed by a pm that records no file lists, os-pm packages are left out of the filesets\n", pmIndexPath, pmFilesPath) + + return nil, true, nil } var list pmFileList if err := json.Unmarshal(body, &list); err != nil { - return nil, fmt.Errorf("parse %s: %w", pmFilesPath, err) + return nil, false, fmt.Errorf("parse %s: %w", pmFilesPath, err) } var filesets []Fileset @@ -59,11 +62,10 @@ func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][] Ecosystem: EcosystemOSPM, Name: name, Version: entry.Version, - Root: "/", Files: files, Symlinks: symlinks, }) } - return filesets, nil + return filesets, false, nil } diff --git a/pkg/sbom/shippedscope/os_pm_fileset_test.go b/pkg/sbom/shippedscope/os_pm_fileset_test.go index 5fa43d486c..3068cc3bd8 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset_test.go +++ b/pkg/sbom/shippedscope/os_pm_fileset_test.go @@ -30,6 +30,7 @@ var _ = Describe("CollectFilesets, os-pm", func() { Expect(git.Files).To(HaveKey("/usr/bin/git")) zlib := filesetOf(filesets, EcosystemOSPM, "zlib", "1.3.1") + Expect(zlib).NotTo(BeNil()) Expect(zlib.Files).To(HaveKey("/usr/lib64/libz.so.1.3.1")) Expect(zlib.Symlinks).To(HaveKeyWithValue("/usr/lib64/libz.so.1", "libz.so.1.3.1")) }) @@ -46,10 +47,63 @@ var _ = Describe("CollectFilesets, os-pm", func() { Expect(err).NotTo(HaveOccurred()) certs := filesetOf(filesets, EcosystemOSPM, "ca-certificates", "20250419") + Expect(certs).NotTo(BeNil()) Expect(certs.Files["/etc/ssl/certs/ca-certificates.crt"].Digest).To(Equal(digestOf("rebuilt after install"))) }) - It("carries on without the file list an older package manager never wrote", func(ctx SpecContext) { + It("leaves out a recorded file the image no longer holds", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/git", "git"), + regularFile("var/lib/pm/files.json", + `{"git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead", "/usr/share/doc/git/README": "sha256:gone"}}}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + git := filesetOf(filesets, EcosystemOSPM, "git", "2.50.1") + Expect(git).NotTo(BeNil()) + Expect(git.Files).To(HaveLen(1)) + }) + + It("reads a file list the image reaches through a hardlink", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/git", "git"), + regularFile("var/lib/pm/files.json.orig", + `{"git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead"}}}`), + hardlink("var/lib/pm/files.json", "var/lib/pm/files.json.orig"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets.OSPMFileListMissing).To(BeFalse()) + Expect(filesetOf(filesets, EcosystemOSPM, "git", "2.50.1")).NotTo(BeNil()) + }) + + It("reads a file list a layer hardlinks to a file another layer wrote", func(ctx SpecContext) { + img, err := imageOf( + []tarEntry{ + regularFile("usr/bin/git", "git"), + regularFile("var/lib/pm/files.json.orig", + `{"git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead"}}}`), + }, + []tarEntry{ + hardlink("var/lib/pm/files.json", "var/lib/pm/files.json.orig"), + }, + ) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets.OSPMFileListMissing).To(BeFalse()) + Expect(filesetOf(filesets, EcosystemOSPM, "git", "2.50.1")).NotTo(BeNil()) + }) + + It("carries on without the file list an older package manager never wrote, and says so", func(ctx SpecContext) { img, err := imageOf([]tarEntry{ regularFile("usr/bin/git", "git"), regularFile("var/lib/pm/index.json", `{"git": {"name": "git", "version": "2.50.1"}}`), @@ -59,7 +113,21 @@ var _ = Describe("CollectFilesets, os-pm", func() { filesets, err := CollectFilesets(ctx, img) Expect(err).NotTo(HaveOccurred()) - Expect(filesets).To(BeEmpty()) + Expect(filesets.Filesets).To(BeEmpty()) + Expect(filesets.OSPMFileListMissing).To(BeTrue()) + }) + + It("does not mistake an image without os-pm packages for one missing its file list", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/bin/git", "git"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesets.Filesets).To(BeEmpty()) + Expect(filesets.OSPMFileListMissing).To(BeFalse()) }) It("reports an image whose file list it cannot read", func(ctx SpecContext) { diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go index 6ee63730d4..d9de460552 100644 --- a/pkg/sbom/shippedscope/python_fileset.go +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -2,6 +2,7 @@ package shippedscope import ( "encoding/csv" + "errors" "io" "path" "strings" @@ -46,7 +47,6 @@ func pythonFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { Ecosystem: EcosystemPython, Name: normalizePythonName(name), Version: version, - Root: distInfo, Files: files, Symlinks: symlinks, }) @@ -63,11 +63,10 @@ func recordPaths(body []byte) []string { var paths []string for { record, err := reader.Read() + if errors.Is(err, io.EOF) { + break + } if err != nil { - if err == io.EOF { - break - } - continue } diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go index a4ac0d09e6..dc663ff22a 100644 --- a/pkg/sbom/shippedscope/python_fileset_test.go +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -46,12 +46,17 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(typing.Files).To(HaveKey("/usr/lib/python3.12/site-packages/typing_extensions.py")) }) - It("credits a compiled file to the source its distribution declares", func(ctx SpecContext) { + It("credits a compiled file to the source its distribution declares, and to no other", func(ctx SpecContext) { img, err := imageOf([]tarEntry{ regularFile("usr/lib/python3.12/site-packages/yaml/__init__.py", "yaml"), regularFile("usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc", "compiled"), + regularFile("usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.opt-1.pyc", "optimized"), regularFile("usr/lib/python3.12/site-packages/PyYAML-6.0.2.dist-info/RECORD", "yaml/__init__.py,sha256=aaa,4\n"), + regularFile("usr/lib/python3.12/site-packages/six.py", "six"), + regularFile("usr/lib/python3.12/site-packages/__pycache__/six.cpython-312.pyc", "compiled six"), + regularFile("usr/lib/python3.12/site-packages/six-1.17.0.dist-info/RECORD", + "six.py,sha256=bbb,3\n"), }) Expect(err).NotTo(HaveOccurred()) @@ -59,7 +64,30 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(err).NotTo(HaveOccurred()) pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml).NotTo(BeNil()) Expect(pyyaml.Files).To(HaveKey("/usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc")) + Expect(pyyaml.Files).To(HaveKey("/usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.opt-1.pyc")) + Expect(pyyaml.Files).NotTo(HaveKey("/usr/lib/python3.12/site-packages/__pycache__/six.cpython-312.pyc")) + + six := filesetOf(filesets, EcosystemPython, "six", "1.17.0") + Expect(six).NotTo(BeNil()) + Expect(six.Files).To(HaveLen(2)) + }) + + It("reads a record whose path is quoted because it holds a comma", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/lib/python3.12/site-packages/odd/a,b.txt", "odd"), + regularFile("usr/lib/python3.12/site-packages/odd-1.0.dist-info/RECORD", + "\"odd/a,b.txt\",sha256=aaa,3\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + odd := filesetOf(filesets, EcosystemPython, "odd", "1.0") + Expect(odd).NotTo(BeNil()) + Expect(odd.Files).To(HaveKey("/usr/lib/python3.12/site-packages/odd/a,b.txt")) }) It("keeps the compiled file when the source it was built from is gone", func(ctx SpecContext) { @@ -74,6 +102,7 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(err).NotTo(HaveOccurred()) pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml).NotTo(BeNil()) Expect(pyyaml.Files).To(HaveLen(1)) Expect(pyyaml.Files).To(HaveKey("/usr/lib/python3.12/site-packages/yaml/__pycache__/__init__.cpython-312.pyc")) }) @@ -91,6 +120,7 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(err).NotTo(HaveOccurred()) pytest := filesetOf(filesets, EcosystemPython, "pytest", "8.4.2") + Expect(pytest).NotTo(BeNil()) Expect(pytest.Files).To(HaveKey("/usr/bin/pytest")) Expect(pytest.Symlinks).To(HaveKeyWithValue("/usr/lib/python3.12/site-packages/pytest/shortcut", "/usr/bin/pytest")) }) @@ -107,6 +137,7 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(err).NotTo(HaveOccurred()) pyyaml := filesetOf(filesets, EcosystemPython, "pyyaml", "6.0.2") + Expect(pyyaml).NotTo(BeNil()) Expect(pyyaml.Files).To(HaveLen(1)) }) }) From d32ff126ae630c5d6185224c9e73b2428da2ccf0 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Thu, 15 Oct 2026 10:22:46 +0300 Subject: [PATCH 4/5] fix(sbom): credit bytecode of a dotted source name to that source, and list filesets in one order Review of #435. The stem of a .pyc was cut at the first dot, so foo.bar.cpython-312.pyc went to foo.py rather than foo.bar.py: one distribution kept a file it never had, another lost a proof. The name is now taken apart from the right, past the interpreter tag and an optional opt-N. Filesets come out sorted by ecosystem, name and version, so a consumer that logs or reports per component sees the same order on every build. A kept body is never nil now, so an empty RECORD yields an empty fileset rather than none. A RECORD line or a rock manifest that does not parse leaves a debug line behind. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/file_index.go | 3 +- pkg/sbom/shippedscope/fileset.go | 16 ++++++- pkg/sbom/shippedscope/fileset_test.go | 42 ++++++++++++++++++ pkg/sbom/shippedscope/javascript_fileset.go | 5 ++- pkg/sbom/shippedscope/lua_fileset.go | 16 +++++-- pkg/sbom/shippedscope/os_pm_fileset.go | 5 ++- pkg/sbom/shippedscope/python_fileset.go | 45 +++++++++++++++++--- pkg/sbom/shippedscope/python_fileset_test.go | 41 ++++++++++++++++++ 8 files changed, 159 insertions(+), 14 deletions(-) create mode 100644 pkg/sbom/shippedscope/fileset_test.go diff --git a/pkg/sbom/shippedscope/file_index.go b/pkg/sbom/shippedscope/file_index.go index e4be119b3f..49182e06b1 100644 --- a/pkg/sbom/shippedscope/file_index.go +++ b/pkg/sbom/shippedscope/file_index.go @@ -133,6 +133,7 @@ func indexImageFiles(ctx context.Context, img v1.Image, capture func(path string return index, captured, nil } +// The body is nil unless kept; a kept body is never nil, however short the file. func readFile(reader io.Reader, header *tar.Header, keepBody bool) (File, []byte, error) { hasher := sha256.New() @@ -154,7 +155,7 @@ func readFile(reader io.Reader, header *tar.Header, keepBody bool) (File, []byte return file, nil, nil } - return file, body.Bytes(), nil + return file, append([]byte{}, body.Bytes()...), nil } // A layer tar spells a hardlink as a regular file plus a link entry beside it, so the bytes diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go index cf7101baa5..b49fe0576c 100644 --- a/pkg/sbom/shippedscope/fileset.go +++ b/pkg/sbom/shippedscope/fileset.go @@ -1,8 +1,10 @@ package shippedscope import ( + "cmp" "context" "path" + "slices" "strings" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -62,9 +64,19 @@ func CollectFilesets(ctx context.Context, img v1.Image) (*Filesets, error) { Filesets: osPM, OSPMFileListMissing: missing, } - filesets.Filesets = append(filesets.Filesets, pythonFilesets(index, manifests)...) + filesets.Filesets = append(filesets.Filesets, pythonFilesets(ctx, index, manifests)...) filesets.Filesets = append(filesets.Filesets, javascriptFilesets(index, manifests)...) - filesets.Filesets = append(filesets.Filesets, luaFilesets(index, manifests)...) + filesets.Filesets = append(filesets.Filesets, luaFilesets(ctx, index, manifests)...) + + // Readers walk maps; a consumer that logs or reports per component should see the same + // order on every build of the same image. + slices.SortStableFunc(filesets.Filesets, func(a, b Fileset) int { + return cmp.Or( + cmp.Compare(a.Ecosystem, b.Ecosystem), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + ) + }) return filesets, nil } diff --git a/pkg/sbom/shippedscope/fileset_test.go b/pkg/sbom/shippedscope/fileset_test.go new file mode 100644 index 0000000000..18718787f5 --- /dev/null +++ b/pkg/sbom/shippedscope/fileset_test.go @@ -0,0 +1,42 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CollectFilesets", func() { + It("lists filesets in one order however the image was read", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("var/lib/pm/files.json", `{ + "zlib": {"version": "1.3.1", "files": {}}, + "git": {"version": "2.50.1", "files": {}}, + "bash": {"version": "5.2", "files": {}} + }`), + regularFile("site/b-1.0.dist-info/RECORD", ""), + regularFile("site/a-1.0.dist-info/RECORD", ""), + regularFile("app/node_modules/y/package.json", `{"name":"y","version":"1.0.0"}`), + regularFile("app/node_modules/x/package.json", `{"name":"x","version":"1.0.0"}`), + regularFile("app/node_modules/x/node_modules/y/package.json", `{"name":"y","version":"0.9.0"}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + var order []string + for _, fileset := range filesets.Filesets { + order = append(order, string(fileset.Ecosystem)+" "+fileset.Name+" "+fileset.Version) + } + Expect(order).To(Equal([]string{ + "javascript x 1.0.0", + "javascript y 0.9.0", + "javascript y 1.0.0", + "os-pm bash 5.2", + "os-pm git 2.50.1", + "os-pm zlib 1.3.1", + "python a 1.0", + "python b 1.0", + })) + }) +}) diff --git a/pkg/sbom/shippedscope/javascript_fileset.go b/pkg/sbom/shippedscope/javascript_fileset.go index 204c6b826d..705c6e2552 100644 --- a/pkg/sbom/shippedscope/javascript_fileset.go +++ b/pkg/sbom/shippedscope/javascript_fileset.go @@ -2,7 +2,9 @@ package shippedscope import ( "encoding/json" + "maps" "path" + "slices" "strings" ) @@ -23,7 +25,8 @@ var distributionNodeDirs = []string{"/usr/share/nodejs", "/usr/lib/nodejs"} func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { var filesets []Fileset - for manifest, body := range manifests { + for _, manifest := range slices.Sorted(maps.Keys(manifests)) { + body := manifests[manifest] if !isNodePackageManifest(manifest) { continue } diff --git a/pkg/sbom/shippedscope/lua_fileset.go b/pkg/sbom/shippedscope/lua_fileset.go index b0513ed443..4c8c6e053f 100644 --- a/pkg/sbom/shippedscope/lua_fileset.go +++ b/pkg/sbom/shippedscope/lua_fileset.go @@ -1,8 +1,13 @@ package shippedscope import ( + "context" + "maps" "path" + "slices" "strings" + + "github.com/werf/logboek" ) const rockManifestName = "rock_manifest" @@ -11,10 +16,11 @@ const rockManifestName = "rock_manifest" // holds little more than the rockspec and its docs: luarocks spreads the code that runs // over trees shared with every other rock, and only the manifest says which files there // are this rock's. -func luaFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { +func luaFilesets(ctx context.Context, index *FileIndex, manifests map[string][]byte) []Fileset { var filesets []Fileset - for manifest, body := range manifests { + for _, manifest := range slices.Sorted(maps.Keys(manifests)) { + body := manifests[manifest] if path.Base(manifest) != rockManifestName { continue } @@ -30,6 +36,8 @@ func luaFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { sections := parseRockManifest(body) if len(sections) == 0 { + logboek.Context(ctx).Debug().LogF("skip %s: not the table luarocks writes\n", manifest) + continue } @@ -43,8 +51,8 @@ func luaFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { for section, entry := range sections { root, ok := roots[section] if !ok { - // Not an install root: the rock's own directory holds it, either as the - // file the section names or as a directory of them. + // Everything else — the rockspec, doc, conf — stays in the rock's own + // directory, as a file or a directory under the section's name. root = path.Join(rockDir, section) } diff --git a/pkg/sbom/shippedscope/os_pm_fileset.go b/pkg/sbom/shippedscope/os_pm_fileset.go index add4c74cba..f97545ac43 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset.go +++ b/pkg/sbom/shippedscope/os_pm_fileset.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "maps" + "slices" "github.com/werf/logboek" ) @@ -47,7 +49,8 @@ func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][] } var filesets []Fileset - for name, entry := range list { + for _, name := range slices.Sorted(maps.Keys(list)) { + entry := list[name] declared := make([]string, 0, len(entry.Files)+len(entry.Symlinks)) for file := range entry.Files { declared = append(declared, rootedPath(file)) diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go index d9de460552..9fa53dd201 100644 --- a/pkg/sbom/shippedscope/python_fileset.go +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -1,11 +1,16 @@ package shippedscope import ( + "context" "encoding/csv" "errors" "io" + "maps" "path" + "slices" "strings" + + "github.com/werf/logboek" ) const distInfoRecordSuffix = ".dist-info/RECORD" @@ -16,12 +21,13 @@ const distInfoRecordSuffix = ".dist-info/RECORD" // directory named after it. // // Names are normalised the PEP 503 way, so a fileset of "PyYAML" is named "pyyaml". -func pythonFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { +func pythonFilesets(ctx context.Context, index *FileIndex, manifests map[string][]byte) []Fileset { var filesets []Fileset pycache := compiledSources(index) - for manifest, body := range manifests { + for _, manifest := range slices.Sorted(maps.Keys(manifests)) { + body := manifests[manifest] if !strings.HasSuffix(manifest, distInfoRecordSuffix) { continue } @@ -35,7 +41,7 @@ func pythonFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { siteDir := path.Dir(distInfo) var declared []string - for _, record := range recordPaths(body) { + for _, record := range recordPaths(ctx, manifest, body) { file := rootedJoin(siteDir, record) declared = append(declared, file) declared = append(declared, pycache[file]...) @@ -55,7 +61,7 @@ func pythonFilesets(index *FileIndex, manifests map[string][]byte) []Fileset { return filesets } -func recordPaths(body []byte) []string { +func recordPaths(ctx context.Context, manifest string, body []byte) []string { reader := csv.NewReader(strings.NewReader(string(body))) reader.FieldsPerRecord = -1 reader.LazyQuotes = true @@ -67,6 +73,8 @@ func recordPaths(body []byte) []string { break } if err != nil { + logboek.Context(ctx).Debug().LogF("skip a line of %s: %s\n", manifest, err) + continue } @@ -96,7 +104,11 @@ func compiledSources(index *FileIndex) map[string][]string { continue } - stem, _, _ := strings.Cut(path.Base(file), ".") + stem, ok := bytecodeStem(path.Base(file)) + if !ok { + continue + } + source := path.Join(path.Dir(dir), stem+".py") compiled[source] = append(compiled[source], file) } @@ -104,6 +116,29 @@ func compiledSources(index *FileIndex) map[string][]string { return compiled } +// bytecodeStem takes ".[.opt-N].pyc" apart from the right, because +// the stem itself may hold dots: "foo.bar.cpython-312.pyc" was compiled from "foo.bar.py". +func bytecodeStem(name string) (string, bool) { + rest := strings.TrimSuffix(name, ".pyc") + + if tagged, _, ok := strings.Cut(rest, ".opt-"); ok { + rest = tagged + } + + stem, _, ok := cutLast(rest, ".") + + return stem, ok && stem != "" +} + +func cutLast(text, separator string) (string, string, bool) { + at := strings.LastIndex(text, separator) + if at < 0 { + return text, "", false + } + + return text[:at], text[at+len(separator):], true +} + func normalizePythonName(name string) string { normalized := strings.Map(func(r rune) rune { if r == '_' || r == '.' { diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go index dc663ff22a..afe6a2a057 100644 --- a/pkg/sbom/shippedscope/python_fileset_test.go +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -74,6 +74,47 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(six.Files).To(HaveLen(2)) }) + It("credits the bytecode of a dotted source name to that source, not to its first component", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("site/foo.py", "foo"), + regularFile("site/foo.bar.py", "foobar"), + regularFile("site/__pycache__/foo.bar.cpython-312.pyc", "compiled foobar"), + regularFile("site/__pycache__/foo.bar.cpython-312.opt-2.pyc", "optimized foobar"), + regularFile("site/__pycache__/foo.pypy310-pp73.pyc", "compiled foo"), + regularFile("site/a-1.0.dist-info/RECORD", "foo.py,,\n"), + regularFile("site/b-1.0.dist-info/RECORD", "foo.bar.py,,\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + a := filesetOf(filesets, EcosystemPython, "a", "1.0") + Expect(a).NotTo(BeNil()) + Expect(a.Files).To(HaveLen(2)) + Expect(a.Files).To(HaveKey("/site/__pycache__/foo.pypy310-pp73.pyc")) + + b := filesetOf(filesets, EcosystemPython, "b", "1.0") + Expect(b).NotTo(BeNil()) + Expect(b.Files).To(HaveLen(3)) + Expect(b.Files).To(HaveKey("/site/__pycache__/foo.bar.cpython-312.pyc")) + Expect(b.Files).To(HaveKey("/site/__pycache__/foo.bar.cpython-312.opt-2.pyc")) + }) + + It("yields an empty fileset for a distribution whose record is empty, not no fileset", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("site/stub-1.0.dist-info/RECORD", ""), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + stub := filesetOf(filesets, EcosystemPython, "stub", "1.0") + Expect(stub).NotTo(BeNil()) + Expect(stub.Files).To(BeEmpty()) + }) + It("reads a record whose path is quoted because it holds a comma", func(ctx SpecContext) { img, err := imageOf([]tarEntry{ regularFile("usr/lib/python3.12/site-packages/odd/a,b.txt", "odd"), From b55aeeea345a136429e2ad1384d9a1bec4779d01 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Thu, 15 Oct 2026 12:37:22 +0300 Subject: [PATCH 5/5] fix(sbom): strip the optimization suffix of a .pyc from the right MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #435. The suffix was cut at the first ".opt-", so bytecode of a source whose own name holds it — my.opt-config.py — lost its stem and was credited to no distribution at all. It is the last dotted component before the interpreter tag, and is now taken as that. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/python_fileset.go | 6 ++++-- pkg/sbom/shippedscope/python_fileset_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go index 9fa53dd201..a93f11c3b2 100644 --- a/pkg/sbom/shippedscope/python_fileset.go +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -117,11 +117,13 @@ func compiledSources(index *FileIndex) map[string][]string { } // bytecodeStem takes ".[.opt-N].pyc" apart from the right, because -// the stem itself may hold dots: "foo.bar.cpython-312.pyc" was compiled from "foo.bar.py". +// the stem itself may hold dots, and may spell anything the suffixes spell: +// "foo.bar.cpython-312.pyc" was compiled from "foo.bar.py", and so was +// "my.opt-config.cpython-312.pyc" from "my.opt-config.py". func bytecodeStem(name string) (string, bool) { rest := strings.TrimSuffix(name, ".pyc") - if tagged, _, ok := strings.Cut(rest, ".opt-"); ok { + if tagged, last, ok := cutLast(rest, "."); ok && strings.HasPrefix(last, "opt-") { rest = tagged } diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go index afe6a2a057..30053a9ab0 100644 --- a/pkg/sbom/shippedscope/python_fileset_test.go +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -81,8 +81,12 @@ var _ = Describe("CollectFilesets, Python", func() { regularFile("site/__pycache__/foo.bar.cpython-312.pyc", "compiled foobar"), regularFile("site/__pycache__/foo.bar.cpython-312.opt-2.pyc", "optimized foobar"), regularFile("site/__pycache__/foo.pypy310-pp73.pyc", "compiled foo"), + regularFile("site/my.opt-config.py", "config"), + regularFile("site/__pycache__/my.opt-config.cpython-312.pyc", "compiled config"), + regularFile("site/__pycache__/my.opt-config.cpython-312.opt-2.pyc", "optimized config"), regularFile("site/a-1.0.dist-info/RECORD", "foo.py,,\n"), regularFile("site/b-1.0.dist-info/RECORD", "foo.bar.py,,\n"), + regularFile("site/c-1.0.dist-info/RECORD", "my.opt-config.py,,\n"), }) Expect(err).NotTo(HaveOccurred()) @@ -99,6 +103,12 @@ var _ = Describe("CollectFilesets, Python", func() { Expect(b.Files).To(HaveLen(3)) Expect(b.Files).To(HaveKey("/site/__pycache__/foo.bar.cpython-312.pyc")) Expect(b.Files).To(HaveKey("/site/__pycache__/foo.bar.cpython-312.opt-2.pyc")) + + c := filesetOf(filesets, EcosystemPython, "c", "1.0") + Expect(c).NotTo(BeNil()) + Expect(c.Files).To(HaveLen(3)) + Expect(c.Files).To(HaveKey("/site/__pycache__/my.opt-config.cpython-312.pyc")) + Expect(c.Files).To(HaveKey("/site/__pycache__/my.opt-config.cpython-312.opt-2.pyc")) }) It("yields an empty fileset for a distribution whose record is empty, not no fileset", func(ctx SpecContext) {