From 3c255c83d3b920a8a6bc09a0d95ca9f0f6580e17 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 09:45:20 +0300 Subject: [PATCH 1/2] feat(sbom): decide which components an image actually ships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shipped-scope has the pieces it needs — an index of the image's files, each component's own files, what every produced binary carries — but nothing yet puts them together into an answer per component. This adds the rule. A component is dropped only when every check misses, from strongest to weakest, each one there for the case a deliberate change to a file defeats the previous: strip, patchelf, a certificate bundle regenerated after install. A Go module or a crate is held by any produced binary that names it. One binary names many, so no exclusivity applies; a binary that could carry components but names none holds all of them; a test binary names nothing. A produced set the build could not establish holds everything, a produced set known to be empty holds nothing — the two were one case in the prototype and collapsing them turns "no go build in this image" into a silent keep. A file-backed component is held by a file of its own in the image, byte for byte, provided no other component of the same ecosystem claims those bytes. Ownership is counted by (name, version), so identical copies of one package are one owner and do not block each other; and within an ecosystem, not across, because a deb shipping an npm tree and the npm packages in it both legitimately own the same files. A component whose every matching file is shared is held only when no co-owner proved itself by a file of its own: the one physical file belongs to whoever did, and this is what tells a stale openssl from the fresh one whose c_rehash it shares. Path and name are looked for only among files no proven component explains, symlinks of proven components included — a symlink carries no bytes and would otherwise hold a stale version through its path. A zero-size file proves nothing. Names every package carries prove nothing either. The rule knows neither where a component's files come from nor which files are produced; it takes occurrences and classifications and returns a decision with the reason and the image paths that made it, so a consumer can record where the component is in this image rather than where it was in the donor. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/match.go | 432 ++++++++++++++++++++++++++++ pkg/sbom/shippedscope/match_test.go | 363 +++++++++++++++++++++++ 2 files changed, 795 insertions(+) create mode 100644 pkg/sbom/shippedscope/match.go create mode 100644 pkg/sbom/shippedscope/match_test.go diff --git a/pkg/sbom/shippedscope/match.go b/pkg/sbom/shippedscope/match.go new file mode 100644 index 0000000000..9a7e5d039c --- /dev/null +++ b/pkg/sbom/shippedscope/match.go @@ -0,0 +1,432 @@ +package shippedscope + +import ( + "context" + "path" + "slices" + + "github.com/werf/logboek" +) + +const ( + EcosystemGo Ecosystem = "go" + EcosystemRust Ecosystem = "rust" +) + +// Occurrence is one file a component's SBOM says the component ships: the path it had in +// the image that recorded it, and the digest of its content there. Nothing says the file +// is still at that path, or still holds those bytes, in the image being matched. +type Occurrence struct { + Path string + // Digest is "sha256:". Empty for a symlink, which carries no bytes of its own. + Digest string + Symlink bool +} + +// Component is what the rule sees of one SBOM component. Occurrences nil means nothing is +// known about the component's files — an external image, an SBOM written before files +// were recorded — and the rule keeps it. Occurrences empty means its files were looked for +// and none were found, which is evidence like any other. +type Component struct { + Ecosystem Ecosystem + Name string + Version string + // Occurrences for a Go module is its import path alone; for a crate, its name and + // version. Neither keeps a file of its own, so neither lists any. + Occurrences []Occurrence +} + +// ProducedBinaries is every binary the image's own build produced, classified. The rule +// takes the whole set rather than one binary at a time because a module is held by any +// binary that carries it, and only the union says which modules no binary does. +// +// A nil *ProducedBinaries means the produced set could not be established — an image +// built outside stapel — and every binary-backed component is kept. An empty one means +// the build produced nothing, which holds no module. +type ProducedBinaries struct { + Binaries []BinaryClassification +} + +type Reason string + +const ( + // ReasonBinary — a produced binary names the component. + ReasonBinary Reason = "binary" + // ReasonContent — a file of the component is in the image, byte for byte, and no other + // component of the same ecosystem claims those bytes. + ReasonContent Reason = "content" + // ReasonContentShared — the component's files in the image are all shared with other + // components of its ecosystem, and none of those proved itself present either. + ReasonContentShared Reason = "content-shared" + // ReasonPath — a path of the component exists in the image with other bytes, and no + // proven component explains that path. + ReasonPath Reason = "path" + // ReasonName — a file named like one of the component's exists somewhere in the image, + // unexplained, and no other component has a file of that name. + ReasonName Reason = "name" + // ReasonNoData — nothing is known about the component's files. + ReasonNoData Reason = "no-data" + // ReasonFailOpen — a produced binary could carry the component but names nothing. + ReasonFailOpen Reason = "fail-open" + // ReasonNotLinked — no produced binary carries the component. + ReasonNotLinked Reason = "not-linked" + // ReasonNoMatch — none of the component's files, paths or names are in the image. + ReasonNoMatch Reason = "no-match" +) + +// Decision says whether one component shipped in the image, and why. +type Decision struct { + Shipped bool + Reason Reason + // Checked counts the occurrences the rule compared against the image; zero when the + // decision did not rest on them. + Checked int + // Matched holds the image paths that proved the component present, so a consumer can + // record where the component is in this image rather than where it was in the donor. + Matched []string +} + +// Match decides, for every component, whether the image ships it. Verdicts come back in +// the order of components. +// +// A component is dropped only when every check misses. Checks run from strongest to +// weakest, each one there for the case the previous was defeated by a deliberate change to +// a file — strip, patchelf, a certificate bundle regenerated after install: +// +// 1. A Go module or a crate is held by any produced binary that names it. One binary +// legitimately names many, so no exclusivity applies. A binary that could carry +// components but names none holds all of them. +// 2. A file-backed component is held by a file of its own in the image, byte for byte — +// provided no other component of the same ecosystem claims the same bytes, counted by +// (name, version) so that identical copies of one package are one owner. A component +// whose every matching file is shared is held only when none of the co-owners proved +// itself by a file of its own: the one physical file belongs to whoever did. +// 3. Among files no proven component explains: a component's path still present in the +// image holds it; failing that, a file named like one of its, when no other component +// has a file of that name and the name is not one every package carries. +// +// A zero-size file proves nothing — its digest is the same for every package. +func Match(ctx context.Context, components []Component, binaries *ProducedBinaries, index *FileIndex) []Decision { + decisions := make([]Decision, len(components)) + + evidence := collectBinaryEvidence(binaries) + owners := collectOwners(components) + names := collectNames(components) + byDigest := indexByDigest(index) + + // A path the rule has already credited to some component. Symlinks of proven + // components count too: a symlink carries no bytes and would otherwise hold a stale + // version through its path. + explained := make(map[string]bool) + type sharedHit struct { + component int + digest string + path string + } + var shared []sharedHit + contentKept := make(map[owner]bool) + + for i, component := range components { + switch component.Ecosystem { + case EcosystemGo: + decisions[i] = evidence.goVerdict(component) + + continue + case EcosystemRust: + decisions[i] = evidence.rustVerdict(component) + + continue + } + + if component.Occurrences == nil { + decisions[i] = Decision{Shipped: true, Reason: ReasonNoData} + + continue + } + + decision := Decision{Checked: len(component.Occurrences)} + for _, occurrence := range component.Occurrences { + if occurrence.Symlink || occurrence.Digest == "" { + continue + } + + for _, imagePath := range byDigest[occurrence.Digest] { + explained[imagePath] = true + + if len(owners[ownerKey{component.Ecosystem, occurrence.Digest}]) > 1 { + shared = append(shared, sharedHit{component: i, digest: occurrence.Digest, path: imagePath}) + + continue + } + + decision.Shipped = true + decision.Reason = ReasonContent + decision.Matched = append(decision.Matched, imagePath) + } + } + + if decision.Shipped { + contentKept[ownerOf(component)] = true + for _, occurrence := range component.Occurrences { + if occurrence.Symlink { + explained[occurrence.Path] = true + } + } + } + + decisions[i] = decision + } + + for _, hit := range shared { + if decisions[hit.component].Shipped { + continue + } + + component := components[hit.component] + if slices.ContainsFunc(owners[ownerKey{component.Ecosystem, hit.digest}], func(other owner) bool { + return other != ownerOf(component) && contentKept[other] + }) { + continue + } + + decisions[hit.component].Shipped = true + decisions[hit.component].Reason = ReasonContentShared + decisions[hit.component].Matched = append(decisions[hit.component].Matched, hit.path) + } + + for i, component := range components { + if decisions[i].Shipped || isBinaryBacked(component.Ecosystem) || component.Occurrences == nil { + continue + } + + decision := &decisions[i] + for _, occurrence := range component.Occurrences { + if explained[occurrence.Path] { + continue + } + if _, ok := index.Files[occurrence.Path]; !ok { + if _, ok := index.Symlinks[occurrence.Path]; !ok { + continue + } + } + + decision.Shipped = true + decision.Reason = ReasonPath + decision.Matched = append(decision.Matched, occurrence.Path) + } + if decision.Shipped { + continue + } + + for _, occurrence := range component.Occurrences { + name := path.Base(occurrence.Path) + if isGenericName(name) || len(names[name]) > 1 { + continue + } + + for imagePath := range index.Files { + if explained[imagePath] || path.Base(imagePath) != name { + continue + } + + decision.Shipped = true + decision.Reason = ReasonName + decision.Matched = append(decision.Matched, imagePath) + } + for imagePath := range index.Symlinks { + if explained[imagePath] || path.Base(imagePath) != name { + continue + } + + decision.Shipped = true + decision.Reason = ReasonName + decision.Matched = append(decision.Matched, imagePath) + } + } + if decision.Shipped { + continue + } + + decision.Reason = ReasonNoMatch + logboek.Context(ctx).Debug().LogF("not shipped %s %s@%s: checked %d files, 0 matches\n", + component.Ecosystem, component.Name, component.Version, decision.Checked) + } + + for i := range decisions { + slices.Sort(decisions[i].Matched) + decisions[i].Matched = slices.Compact(decisions[i].Matched) + } + + return decisions +} + +// indexByDigest inverts the index for the content pass, leaving out zero-size files: their +// digest is the same for every package and proves nothing. +func indexByDigest(index *FileIndex) map[string][]string { + byDigest := make(map[string][]string) + for imagePath, file := range index.Files { + if file.Size == 0 { + continue + } + + byDigest[file.Digest] = append(byDigest[file.Digest], imagePath) + } + for _, paths := range byDigest { + slices.Sort(paths) + } + + return byDigest +} + +func isBinaryBacked(ecosystem Ecosystem) bool { + return ecosystem == EcosystemGo || ecosystem == EcosystemRust +} + +type binaryEvidence struct { + known bool + + goFailOpen bool + modules map[string]bool + + rustFailOpen bool + crates map[Crate]bool +} + +func collectBinaryEvidence(binaries *ProducedBinaries) binaryEvidence { + evidence := binaryEvidence{ + modules: make(map[string]bool), + crates: make(map[Crate]bool), + } + if binaries == nil { + return evidence + } + evidence.known = true + + for _, binary := range binaries.Binaries { + if binary.TestBinary { + continue + } + + switch binary.Go { + case VerdictUnknown: + evidence.goFailOpen = true + case VerdictNames: + for _, module := range binary.Modules { + evidence.modules[module] = true + } + } + + switch binary.Rust { + case VerdictUnknown: + evidence.rustFailOpen = true + case VerdictNames: + for _, crate := range binary.Crates { + evidence.crates[crate] = true + } + } + } + + return evidence +} + +func (e binaryEvidence) goVerdict(component Component) Decision { + switch { + case !e.known: + return Decision{Shipped: true, Reason: ReasonNoData} + case e.goFailOpen: + return Decision{Shipped: true, Reason: ReasonFailOpen} + case e.modules[component.Name]: + return Decision{Shipped: true, Reason: ReasonBinary} + } + + return Decision{Reason: ReasonNotLinked} +} + +func (e binaryEvidence) rustVerdict(component Component) Decision { + switch { + case !e.known: + return Decision{Shipped: true, Reason: ReasonNoData} + case e.rustFailOpen: + return Decision{Shipped: true, Reason: ReasonFailOpen} + case e.crates[Crate{Name: component.Name, Version: component.Version}]: + return Decision{Shipped: true, Reason: ReasonBinary} + } + + return Decision{Reason: ReasonNotLinked} +} + +// owner is a component as the exclusivity count sees it: one package version, however +// many copies of it the image holds. +type owner struct { + ecosystem Ecosystem + name string + version string +} + +func ownerOf(component Component) owner { + return owner{component.Ecosystem, component.Name, component.Version} +} + +type ownerKey struct { + ecosystem Ecosystem + digest string +} + +// collectOwners answers, for every digest a file-backed component claims, which owners of +// its own ecosystem claim it. The ecosystem is part of the key because ownership across +// ecosystems is hierarchical, not competing: a deb package that ships an npm tree and the +// npm packages in that tree both legitimately own the same files. +func collectOwners(components []Component) map[ownerKey][]owner { + owners := make(map[ownerKey][]owner) + for _, component := range components { + if isBinaryBacked(component.Ecosystem) { + continue + } + + for _, occurrence := range component.Occurrences { + if occurrence.Symlink || occurrence.Digest == "" { + continue + } + + key := ownerKey{component.Ecosystem, occurrence.Digest} + if !slices.Contains(owners[key], ownerOf(component)) { + owners[key] = append(owners[key], ownerOf(component)) + } + } + } + + return owners +} + +func collectNames(components []Component) map[string][]owner { + names := make(map[string][]owner) + for _, component := range components { + if isBinaryBacked(component.Ecosystem) { + continue + } + + for _, occurrence := range component.Occurrences { + name := path.Base(occurrence.Path) + if !slices.Contains(names[name], ownerOf(component)) { + names[name] = append(names[name], ownerOf(component)) + } + } + } + + return names +} + +// isGenericName names the files nearly every package carries, so that one of them in the +// image says nothing about which package it came with. +func isGenericName(name string) bool { + switch name { + case "LICENSE", "LICENSE.txt", "LICENSE.md", "COPYING", "COPYING.LIB", "NOTICE", + "README", "README.md", "README.txt", "README.rst", "CHANGELOG", "CHANGELOG.md", "changelog", + "changelog.gz", "changelog.Debian.gz", "copyright", "AUTHORS", "CONTRIBUTING.md", + "package.json", "RECORD", "METADATA", "WHEEL", "INSTALLER", "top_level.txt", "__init__.py", + "index.js", "index.d.ts", "setup.py", "Makefile", ".gitignore", ".npmignore": + return true + } + + return false +} diff --git a/pkg/sbom/shippedscope/match_test.go b/pkg/sbom/shippedscope/match_test.go new file mode 100644 index 0000000000..09d2fbd6f5 --- /dev/null +++ b/pkg/sbom/shippedscope/match_test.go @@ -0,0 +1,363 @@ +package shippedscope + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Match, binary-backed components", func() { + goBinaryNaming := func(modules ...string) BinaryClassification { + return BinaryClassification{Go: VerdictNames, Modules: modules, Rust: VerdictNone} + } + + It("holds the modules a produced binary names and drops the rest", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "gopkg.in/yaml.v3"}, + {Ecosystem: EcosystemGo, Name: "github.com/spf13/cobra"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{goBinaryNaming("gopkg.in/yaml.v3")}}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNotLinked})) + }) + + It("lets one binary hold several modules at once", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + {Ecosystem: EcosystemGo, Name: "b"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{goBinaryNaming("a", "b")}}, emptyIndex()) + + Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonBinary}))) + }) + + It("holds every module when any produced binary could carry modules but names none", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{ + goBinaryNaming("b"), + {Go: VerdictUnknown, Rust: VerdictNone}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonFailOpen})) + }) + + It("lets a test binary speak for nobody, whatever it names", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "github.com/stretchr/testify"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{ + {TestBinary: true, Go: VerdictNames, Modules: []string{"github.com/stretchr/testify"}}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Reason: ReasonNotLinked})) + }) + + It("does not let a test binary that cannot be read fail everything open", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{ + {TestBinary: true, Go: VerdictUnknown, Rust: VerdictUnknown}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Reason: ReasonNotLinked})) + }) + + It("drops every module when the build is known to have produced no binary", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + }, &ProducedBinaries{}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Reason: ReasonNotLinked})) + }) + + It("holds every module when the produced set could not be established at all", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.210"}, + }, nil, emptyIndex()) + + Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonNoData}))) + }) + + It("matches a crate by name and version, and a module by path alone", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.210"}, + {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.100"}, + {Ecosystem: EcosystemGo, Name: "a", Version: "v9.9.9"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{ + {Go: VerdictNames, Modules: []string{"a"}, Rust: VerdictNames, Crates: []Crate{{Name: "serde", Version: "1.0.210"}}}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNotLinked})) + Expect(decisions[2]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) + }) + + It("reads Go and Rust evidence independently of each other", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + {Ecosystem: EcosystemRust, Name: "serde", Version: "1"}, + }, &ProducedBinaries{Binaries: []BinaryClassification{ + {Go: VerdictNames, Modules: []string{"b"}, Rust: VerdictUnknown}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Reason: ReasonNotLinked})) + Expect(decisions[1]).To(Equal(Decision{Shipped: true, Reason: ReasonFailOpen})) + }) +}) + +var _ = Describe("Match, file-backed components", func() { + noBinaries := &ProducedBinaries{} + + It("holds a component whose own file is in the image byte for byte", func(ctx SpecContext) { + index := indexOf(map[string]File{"/bin/jq": {Digest: "sha256:jq", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "jq", Version: "1.6", Occurrences: []Occurrence{{Path: "/usr/bin/jq", Digest: "sha256:jq"}}}, + {Ecosystem: EcosystemOSPM, Name: "gcc-12", Version: "12", Occurrences: []Occurrence{{Path: "/usr/bin/gcc", Digest: "sha256:gcc"}}}, + }, noBinaries, index) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 1, Matched: []string{"/bin/jq"}})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 1})) + }) + + It("keeps a component nothing is known about, and drops one whose files were looked for and not found", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "libfoo", Version: "1"}, + {Ecosystem: EcosystemOSPM, Name: "libbar", Version: "1", Occurrences: []Occurrence{}}, + }, noBinaries, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonNoData})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch})) + }) + + It("does not take a zero-size file as evidence", func(ctx SpecContext) { + index := indexOf(map[string]File{"/x/__init__.py": {Digest: "sha256:empty", Size: 0}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemPython, Name: "x", Version: "1", Occurrences: []Occurrence{{Path: "/site/x/__init__.py", Digest: "sha256:empty"}}}, + }, noBinaries, index) + + Expect(decisions[0].Shipped).To(BeFalse()) + }) + + It("credits a shared file to the owner that proved itself by a file of its own", func(ctx SpecContext) { + index := indexOf(map[string]File{ + "/usr/bin/c_rehash": {Digest: "sha256:rehash", Size: 10}, + "/usr/lib/libssl.so.3": {Digest: "sha256:ssl-new", Size: 10}, + }) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "openssl", Version: "3.0.13", Occurrences: []Occurrence{ + {Path: "/usr/bin/c_rehash", Digest: "sha256:rehash"}, + {Path: "/usr/lib/libssl.so.3", Digest: "sha256:ssl-new"}, + }}, + {Ecosystem: EcosystemOSPM, Name: "openssl", Version: "3.0.11", Occurrences: []Occurrence{ + {Path: "/usr/bin/c_rehash", Digest: "sha256:rehash"}, + {Path: "/usr/lib/libssl.so.3", Digest: "sha256:ssl-old"}, + }}, + }, noBinaries, index) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 2, Matched: []string{"/usr/lib/libssl.so.3"}})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 2})) + }) + + It("holds both owners of a shared file when neither proved itself otherwise", func(ctx SpecContext) { + index := indexOf(map[string]File{"/usr/share/x/data": {Digest: "sha256:data", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "foo", Version: "1", Occurrences: []Occurrence{{Path: "/usr/share/x/data", Digest: "sha256:data"}}}, + {Ecosystem: EcosystemOSPM, Name: "foo-data", Version: "1", Occurrences: []Occurrence{{Path: "/usr/share/x/data", Digest: "sha256:data"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonContentShared, Checked: 1, Matched: []string{"/usr/share/x/data"}}))) + }) + + It("counts ownership within an ecosystem, so a deb shipping an npm tree does not block the npm packages", func(ctx SpecContext) { + index := indexOf(map[string]File{"/usr/share/nodejs/lodash/lodash.js": {Digest: "sha256:lodash", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "node-lodash", Version: "4", Occurrences: []Occurrence{{Path: "/usr/share/nodejs/lodash/lodash.js", Digest: "sha256:lodash"}}}, + {Ecosystem: EcosystemJavaScript, Name: "lodash", Version: "4.17.21", Occurrences: []Occurrence{{Path: "/usr/share/nodejs/lodash/lodash.js", Digest: "sha256:lodash"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(HaveField("Reason", ReasonContent))) + }) + + It("counts ownership by package version, so identical copies of one package are one owner", func(ctx SpecContext) { + index := indexOf(map[string]File{"/app/node_modules/debug/index.js": {Digest: "sha256:debug", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "2.6.9", Occurrences: []Occurrence{{Path: "/app/node_modules/debug/index.js", Digest: "sha256:debug"}}}, + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "2.6.9", Occurrences: []Occurrence{{Path: "/app/node_modules/x/node_modules/debug/index.js", Digest: "sha256:debug"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(HaveField("Reason", ReasonContent))) + }) + + It("holds a component whose file was rewritten in place, by its path", func(ctx SpecContext) { + index := indexOf(map[string]File{"/etc/ssl/certs/ca-certificates.crt": {Digest: "sha256:regenerated", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "ca-certificates", Version: "2024", Occurrences: []Occurrence{{Path: "/etc/ssl/certs/ca-certificates.crt", Digest: "sha256:as-installed"}}}, + }, noBinaries, index) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonPath, Checked: 1, Matched: []string{"/etc/ssl/certs/ca-certificates.crt"}})) + }) + + It("does not let a path explained by a proven component hold a stale one", func(ctx SpecContext) { + index := indexOf(map[string]File{"/usr/lib/libc.so.6": {Digest: "sha256:libc-new", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "libc6", Version: "2.36", Occurrences: []Occurrence{{Path: "/usr/lib/libc.so.6", Digest: "sha256:libc-new"}}}, + {Ecosystem: EcosystemOSPM, Name: "libc6", Version: "2.31", Occurrences: []Occurrence{{Path: "/usr/lib/libc.so.6", Digest: "sha256:libc-old"}}}, + }, noBinaries, index) + + Expect(decisions[0].Reason).To(Equal(ReasonContent)) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 1})) + }) + + It("does not let a proven component's symlink hold a stale version by path", func(ctx SpecContext) { + index := &FileIndex{ + Files: map[string]File{"/usr/lib/libz.so.1.3": {Digest: "sha256:z-new", Size: 10}}, + Symlinks: map[string]string{"/usr/lib/libz.so.1": "libz.so.1.3"}, + } + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "zlib1g", Version: "1.3", Occurrences: []Occurrence{ + {Path: "/usr/lib/libz.so.1.3", Digest: "sha256:z-new"}, + {Path: "/usr/lib/libz.so.1", Symlink: true}, + }}, + {Ecosystem: EcosystemOSPM, Name: "zlib1g", Version: "1.2", Occurrences: []Occurrence{ + {Path: "/usr/lib/libz.so.1.2", Digest: "sha256:z-old"}, + {Path: "/usr/lib/libz.so.1", Symlink: true}, + }}, + }, noBinaries, index) + + Expect(decisions[0].Reason).To(Equal(ReasonContent)) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 2})) + }) + + It("holds a relocated and rewritten file by its unique name", func(ctx SpecContext) { + index := indexOf(map[string]File{"/opt/tool/bin/jq": {Digest: "sha256:jq-stripped", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "jq", Version: "1.6", Occurrences: []Occurrence{{Path: "/usr/bin/jq", Digest: "sha256:jq"}}}, + }, noBinaries, index) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonName, Checked: 1, Matched: []string{"/opt/tool/bin/jq"}})) + }) + + It("does not match by a name every package carries, nor by one two components share", func(ctx SpecContext) { + index := indexOf(map[string]File{ + "/opt/a/LICENSE": {Digest: "sha256:lic", Size: 10}, + "/opt/a/utils.py": {Digest: "sha256:u", Size: 10}, + }) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemPython, Name: "a", Version: "1", Occurrences: []Occurrence{{Path: "/site/a/LICENSE", Digest: "sha256:lic-a"}, {Path: "/site/a/utils.py", Digest: "sha256:ua"}}}, + {Ecosystem: EcosystemPython, Name: "b", Version: "1", Occurrences: []Occurrence{{Path: "/site/b/utils.py", Digest: "sha256:ub"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(HaveField("Shipped", BeFalse()))) + }) + + It("looks for a path or a name only among files no proven component explains", func(ctx SpecContext) { + index := indexOf(map[string]File{"/usr/bin/tool": {Digest: "sha256:tool-new", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "tool", Version: "2", Occurrences: []Occurrence{{Path: "/usr/bin/tool", Digest: "sha256:tool-new"}}}, + {Ecosystem: EcosystemOSPM, Name: "tool-legacy", Version: "1", Occurrences: []Occurrence{{Path: "/opt/legacy/tool", Digest: "sha256:tool-old"}}}, + }, noBinaries, index) + + Expect(decisions[0].Reason).To(Equal(ReasonContent)) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 1})) + }) + + It("returns decisions in the order of components", func(ctx SpecContext) { + index := indexOf(map[string]File{"/bin/jq": {Digest: "sha256:jq", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "gcc", Version: "12", Occurrences: []Occurrence{{Path: "/usr/bin/gcc", Digest: "sha256:gcc"}}}, + {Ecosystem: EcosystemGo, Name: "a"}, + {Ecosystem: EcosystemOSPM, Name: "jq", Version: "1.6", Occurrences: []Occurrence{{Path: "/usr/bin/jq", Digest: "sha256:jq"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveLen(3)) + Expect(decisions[0].Shipped).To(BeFalse()) + Expect(decisions[1].Shipped).To(BeFalse()) + Expect(decisions[2].Shipped).To(BeTrue()) + }) +}) + +// The end-to-end stand of the concept (bench-e2e/expected.json), as the rule sees it: a +// builder's components with the files the builder gave them, matched against what the +// final image took. +var _ = Describe("Match, the concept's end-to-end stand", func() { + It("reproduces the expected decisions", func(ctx SpecContext) { + final := indexOf(map[string]File{ + "/app": {Digest: "sha256:app", Size: 10}, + "/app/py/requests/api.py": {Digest: "sha256:req-api", Size: 10}, + "/app/js/lodash/lodash.js": {Digest: "sha256:lodash", Size: 10}, + "/bin/jq": {Digest: "sha256:jq", Size: 10}, + "/lib/x86_64/libz.so.1.2.13": {Digest: "sha256:libz", Size: 10}, + "/lib/x86_64/libc.so.6": {Digest: "sha256:libc", Size: 10}, + }) + + produced := &ProducedBinaries{Binaries: []BinaryClassification{ + {Go: VerdictNames, Modules: []string{"gopkg.in/yaml.v3"}, Rust: VerdictNone}, + }} + + components := []Component{ + {Ecosystem: EcosystemGo, Name: "gopkg.in/yaml.v3"}, + {Ecosystem: EcosystemGo, Name: "github.com/spf13/cobra"}, + {Ecosystem: EcosystemGo, Name: "github.com/spf13/pflag"}, + {Ecosystem: EcosystemPython, Name: "requests", Version: "2.32", Occurrences: []Occurrence{{Path: "/venv/lib/site-packages/requests/api.py", Digest: "sha256:req-api"}}}, + {Ecosystem: EcosystemPython, Name: "certifi", Version: "2024", Occurrences: []Occurrence{{Path: "/venv/lib/site-packages/certifi/core.py", Digest: "sha256:certifi"}}}, + {Ecosystem: EcosystemPython, Name: "idna", Version: "3", Occurrences: []Occurrence{{Path: "/venv/lib/site-packages/idna/core.py", Digest: "sha256:idna"}}}, + {Ecosystem: EcosystemPython, Name: "pytest", Version: "8", Occurrences: []Occurrence{{Path: "/venv/lib/site-packages/pytest/main.py", Digest: "sha256:pytest"}}}, + {Ecosystem: EcosystemJavaScript, Name: "lodash", Version: "4.17.21", Occurrences: []Occurrence{{Path: "/src/node_modules/lodash/lodash.js", Digest: "sha256:lodash"}}}, + {Ecosystem: EcosystemJavaScript, Name: "jest", Version: "29", Occurrences: []Occurrence{{Path: "/src/node_modules/jest/bin/jest.js", Digest: "sha256:jest"}}}, + {Ecosystem: EcosystemJavaScript, Name: "@babel/cli", Version: "7", Occurrences: []Occurrence{{Path: "/usr/share/nodejs/@babel/cli/bin/babel.js", Digest: "sha256:babel"}}}, + {Ecosystem: EcosystemOSPM, Name: "jq", Version: "1.6", Occurrences: []Occurrence{{Path: "/usr/bin/jq", Digest: "sha256:jq"}}}, + {Ecosystem: EcosystemOSPM, Name: "libpcre3", Version: "8.39", Occurrences: []Occurrence{{Path: "/lib/x86_64/libpcre.so.3.13", Digest: "sha256:pcre-builder"}}}, + {Ecosystem: EcosystemOSPM, Name: "zlib1g", Version: "1.2.13", Occurrences: []Occurrence{{Path: "/lib/x86_64/libz.so.1.2.13", Digest: "sha256:libz"}}}, + {Ecosystem: EcosystemOSPM, Name: "libc6", Version: "2.36", Occurrences: []Occurrence{{Path: "/lib/x86_64/libc.so.6", Digest: "sha256:libc"}}}, + {Ecosystem: EcosystemOSPM, Name: "gcc-12", Version: "12", Occurrences: []Occurrence{{Path: "/usr/bin/gcc-12", Digest: "sha256:gcc"}}}, + {Ecosystem: EcosystemOSPM, Name: "python3-pip", Version: "23", Occurrences: []Occurrence{{Path: "/usr/bin/pip3", Digest: "sha256:pip"}}}, + {Ecosystem: EcosystemOSPM, Name: "npm", Version: "9", Occurrences: []Occurrence{{Path: "/usr/bin/npm", Digest: "sha256:npm"}}}, + } + + decisions := Match(ctx, components, produced, final) + + shipped := map[string]bool{} + for i, component := range components { + shipped[string(component.Ecosystem)+":"+component.Name] = decisions[i].Shipped + } + Expect(shipped).To(Equal(map[string]bool{ + "go:gopkg.in/yaml.v3": true, + "go:github.com/spf13/cobra": false, + "go:github.com/spf13/pflag": false, + "python:requests": true, + "python:certifi": false, + "python:idna": false, + "python:pytest": false, + "javascript:lodash": true, + "javascript:jest": false, + "javascript:@babel/cli": false, + "os-pm:jq": true, + "os-pm:libpcre3": false, + "os-pm:zlib1g": true, + "os-pm:libc6": true, + "os-pm:gcc-12": false, + "os-pm:python3-pip": false, + "os-pm:npm": false, + })) + }) +}) + +func emptyIndex() *FileIndex { + return &FileIndex{Files: map[string]File{}, Symlinks: map[string]string{}} +} + +func indexOf(files map[string]File) *FileIndex { + return &FileIndex{Files: files, Symlinks: map[string]string{}} +} From 9536952bf051363ac3caa71731fcbd715b1ebb24 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 10:15:53 +0300 Subject: [PATCH 2/2] fix(sbom): treat same-bytes files at two paths as two files in the shipped-scope rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass 1b keyed ownership by digest alone, so two components each shipping a byte-identical file at its own path contested one another: the one without a second unique file was dropped while its own file sat in the image, and the other was credited with a path it never declared. A component's image path is now contested only by the components that declare that path; every owner of the bytes contests it only when nobody declares it, i.e. the file was moved on its way in. Also from review: the path tier's relocation contract is stated on Occurrence (the caller relocates add→to, the rule compares verbatim); a proven owner and a content-shared component are credited with every shared file in Matched, not only the first; a module a binary names reads `binary` even when another binary is unreadable; the name tier uses a basename index instead of walking the image per occurrence; ProducedBinaries carries a Known flag instead of a nil sentinel; Checked counts only files whose bytes were looked for. Specs pin each branch and the stand asserts reasons, not just the verdict. Signed-off-by: Radmir Khurum --- pkg/sbom/shippedscope/match.go | 166 ++++++++++++++++++-------- pkg/sbom/shippedscope/match_test.go | 177 ++++++++++++++++++++++------ 2 files changed, 260 insertions(+), 83 deletions(-) diff --git a/pkg/sbom/shippedscope/match.go b/pkg/sbom/shippedscope/match.go index 9a7e5d039c..f5bb760179 100644 --- a/pkg/sbom/shippedscope/match.go +++ b/pkg/sbom/shippedscope/match.go @@ -5,6 +5,8 @@ import ( "path" "slices" + "github.com/samber/lo" + "github.com/werf/logboek" ) @@ -13,9 +15,14 @@ const ( EcosystemRust Ecosystem = "rust" ) -// Occurrence is one file a component's SBOM says the component ships: the path it had in -// the image that recorded it, and the digest of its content there. Nothing says the file -// is still at that path, or still holds those bytes, in the image being matched. +// Occurrence is one file a component's SBOM says the component ships: the path it has in +// the image being matched, and the digest its content had in the image that recorded it. +// Nothing says the file is still at that path, or still holds those bytes. +// +// The path is the caller's to relocate: an occurrence recorded in a donor under +// `/usr/bin/jq` and imported with `to: /bin` arrives here as `/bin/jq`. The rule compares +// paths verbatim and knows nothing of imports, so a donor path passed unrelocated makes +// the path tier miss silently while the content tier still matches. type Occurrence struct { Path string // Digest is "sha256:". Empty for a symlink, which carries no bytes of its own. @@ -31,8 +38,8 @@ type Component struct { Ecosystem Ecosystem Name string Version string - // Occurrences for a Go module is its import path alone; for a crate, its name and - // version. Neither keeps a file of its own, so neither lists any. + // Occurrences is nil for a Go module or a crate: neither keeps a file of its own, and + // both are proven by the produced binaries instead. Occurrences []Occurrence } @@ -40,10 +47,16 @@ type Component struct { // takes the whole set rather than one binary at a time because a module is held by any // binary that carries it, and only the union says which modules no binary does. // -// A nil *ProducedBinaries means the produced set could not be established — an image -// built outside stapel — and every binary-backed component is kept. An empty one means +// Known false means the produced set could not be established — an image built outside +// stapel — and every binary-backed component is kept. Known true with no binaries means // the build produced nothing, which holds no module. +// +// A binary imported from another image is not produced here and does not belong in this +// set: the modules it carries are the donor's SBOM's to prove, and arrive as components +// with their own evidence. Putting it here would hold the donor's modules in an image +// that merely copied the file. type ProducedBinaries struct { + Known bool Binaries []BinaryClassification } @@ -78,8 +91,8 @@ const ( type Decision struct { Shipped bool Reason Reason - // Checked counts the occurrences the rule compared against the image; zero when the - // decision did not rest on them. + // Checked counts the files whose bytes the rule looked for in the image; symlinks + // carry none and are not counted. Zero when the decision did not rest on files. Checked int // Matched holds the image paths that proved the component present, so a consumer can // record where the component is in this image rather than where it was in the donor. @@ -106,13 +119,27 @@ type Decision struct { // has a file of that name and the name is not one every package carries. // // A zero-size file proves nothing — its digest is the same for every package. -func Match(ctx context.Context, components []Component, binaries *ProducedBinaries, index *FileIndex) []Decision { +// +// Two files with the same bytes at two paths are two files. A component's own path in the +// image is contested only by the components that declare that path; the bytes it shares +// with a file elsewhere are not a claim on that file. Only when nobody declares the image +// path — the file was moved on its way in — do all owners of the bytes contest it. +// +// Binary-backed and file-backed components do not see each other: a produced Go binary is +// a file in the image like any other and can hold an unrelated file-backed component of +// the same name through the name tier, which is the holding direction. +func Match(ctx context.Context, components []Component, binaries ProducedBinaries, index *FileIndex) []Decision { + if index == nil { + panic("shippedscope.Match: index is nil") + } + decisions := make([]Decision, len(components)) evidence := collectBinaryEvidence(binaries) - owners := collectOwners(components) + owners, ownersAtPath := collectOwners(components) names := collectNames(components) byDigest := indexByDigest(index) + byName := indexByName(index) // A path the rule has already credited to some component. Symlinks of proven // components count too: a symlink carries no bytes and would otherwise hold a stale @@ -144,16 +171,27 @@ func Match(ctx context.Context, components []Component, binaries *ProducedBinari continue } - decision := Decision{Checked: len(component.Occurrences)} + decision := Decision{} for _, occurrence := range component.Occurrences { if occurrence.Symlink || occurrence.Digest == "" { continue } + decision.Checked++ for _, imagePath := range byDigest[occurrence.Digest] { explained[imagePath] = true - if len(owners[ownerKey{component.Ecosystem, occurrence.Digest}]) > 1 { + contestants := ownersAtPath[ownerPathKey{component.Ecosystem, occurrence.Digest, imagePath}] + if len(contestants) == 0 { + contestants = owners[ownerKey{component.Ecosystem, occurrence.Digest}] + } + if !lo.Contains(contestants, ownerOf(component)) { + // Someone else declares this path with these bytes; the component's own + // claim is on another path, present or not. + continue + } + + if len(contestants) > 1 { shared = append(shared, sharedHit{component: i, digest: occurrence.Digest, path: imagePath}) continue @@ -178,20 +216,30 @@ func Match(ctx context.Context, components []Component, binaries *ProducedBinari } for _, hit := range shared { - if decisions[hit.component].Shipped { - continue - } - component := components[hit.component] - if slices.ContainsFunc(owners[ownerKey{component.Ecosystem, hit.digest}], func(other owner) bool { + decision := &decisions[hit.component] + + contestants := ownersAtPath[ownerPathKey{component.Ecosystem, hit.digest, hit.path}] + if len(contestants) == 0 { + contestants = owners[ownerKey{component.Ecosystem, hit.digest}] + } + provenElsewhere := lo.ContainsBy(contestants, func(other owner) bool { return other != ownerOf(component) && contentKept[other] - }) { + }) + + switch { + case decision.Reason == ReasonContent: + // A proven owner is credited with its shared files too: the rule has decided + // they are its, and a consumer rewriting occurrences from Matched would + // otherwise lose them for the next image in the chain. + decision.Matched = append(decision.Matched, hit.path) + case provenElsewhere: continue + default: + decision.Shipped = true + decision.Reason = ReasonContentShared + decision.Matched = append(decision.Matched, hit.path) } - - decisions[hit.component].Shipped = true - decisions[hit.component].Reason = ReasonContentShared - decisions[hit.component].Matched = append(decisions[hit.component].Matched, hit.path) } for i, component := range components { @@ -224,17 +272,8 @@ func Match(ctx context.Context, components []Component, binaries *ProducedBinari continue } - for imagePath := range index.Files { - if explained[imagePath] || path.Base(imagePath) != name { - continue - } - - decision.Shipped = true - decision.Reason = ReasonName - decision.Matched = append(decision.Matched, imagePath) - } - for imagePath := range index.Symlinks { - if explained[imagePath] || path.Base(imagePath) != name { + for _, imagePath := range byName[name] { + if explained[imagePath] { continue } @@ -278,6 +317,21 @@ func indexByDigest(index *FileIndex) map[string][]string { return byDigest } +func indexByName(index *FileIndex) map[string][]string { + byName := make(map[string][]string) + for imagePath := range index.Files { + byName[path.Base(imagePath)] = append(byName[path.Base(imagePath)], imagePath) + } + for imagePath := range index.Symlinks { + byName[path.Base(imagePath)] = append(byName[path.Base(imagePath)], imagePath) + } + for _, paths := range byName { + slices.Sort(paths) + } + + return byName +} + func isBinaryBacked(ecosystem Ecosystem) bool { return ecosystem == EcosystemGo || ecosystem == EcosystemRust } @@ -292,15 +346,12 @@ type binaryEvidence struct { crates map[Crate]bool } -func collectBinaryEvidence(binaries *ProducedBinaries) binaryEvidence { +func collectBinaryEvidence(binaries ProducedBinaries) binaryEvidence { evidence := binaryEvidence{ + known: binaries.Known, modules: make(map[string]bool), crates: make(map[Crate]bool), } - if binaries == nil { - return evidence - } - evidence.known = true for _, binary := range binaries.Binaries { if binary.TestBinary { @@ -333,10 +384,10 @@ func (e binaryEvidence) goVerdict(component Component) Decision { switch { case !e.known: return Decision{Shipped: true, Reason: ReasonNoData} - case e.goFailOpen: - return Decision{Shipped: true, Reason: ReasonFailOpen} case e.modules[component.Name]: return Decision{Shipped: true, Reason: ReasonBinary} + case e.goFailOpen: + return Decision{Shipped: true, Reason: ReasonFailOpen} } return Decision{Reason: ReasonNotLinked} @@ -346,10 +397,10 @@ func (e binaryEvidence) rustVerdict(component Component) Decision { switch { case !e.known: return Decision{Shipped: true, Reason: ReasonNoData} - case e.rustFailOpen: - return Decision{Shipped: true, Reason: ReasonFailOpen} case e.crates[Crate{Name: component.Name, Version: component.Version}]: return Decision{Shipped: true, Reason: ReasonBinary} + case e.rustFailOpen: + return Decision{Shipped: true, Reason: ReasonFailOpen} } return Decision{Reason: ReasonNotLinked} @@ -372,12 +423,20 @@ type ownerKey struct { digest string } +type ownerPathKey struct { + ecosystem Ecosystem + digest string + path string +} + // collectOwners answers, for every digest a file-backed component claims, which owners of -// its own ecosystem claim it. The ecosystem is part of the key because ownership across -// ecosystems is hierarchical, not competing: a deb package that ships an npm tree and the -// npm packages in that tree both legitimately own the same files. -func collectOwners(components []Component) map[ownerKey][]owner { +// its own ecosystem claim it — in all, and at each declared path. The ecosystem is part of +// the key because ownership across ecosystems is hierarchical, not competing: a deb +// package that ships an npm tree and the npm packages in that tree both legitimately own +// the same files. +func collectOwners(components []Component) (map[ownerKey][]owner, map[ownerPathKey][]owner) { owners := make(map[ownerKey][]owner) + ownersAtPath := make(map[ownerPathKey][]owner) for _, component := range components { if isBinaryBacked(component.Ecosystem) { continue @@ -389,15 +448,22 @@ func collectOwners(components []Component) map[ownerKey][]owner { } key := ownerKey{component.Ecosystem, occurrence.Digest} - if !slices.Contains(owners[key], ownerOf(component)) { + if !lo.Contains(owners[key], ownerOf(component)) { owners[key] = append(owners[key], ownerOf(component)) } + pathKey := ownerPathKey{component.Ecosystem, occurrence.Digest, occurrence.Path} + if !lo.Contains(ownersAtPath[pathKey], ownerOf(component)) { + ownersAtPath[pathKey] = append(ownersAtPath[pathKey], ownerOf(component)) + } } } - return owners + return owners, ownersAtPath } +// collectNames answers which owners declare a file of each basename. Unlike content +// ownership this is counted across ecosystems: a name is the weakest evidence, and a file +// two components of any kind could have left there says nothing about which one did. func collectNames(components []Component) map[string][]owner { names := make(map[string][]owner) for _, component := range components { @@ -407,7 +473,7 @@ func collectNames(components []Component) map[string][]owner { for _, occurrence := range component.Occurrences { name := path.Base(occurrence.Path) - if !slices.Contains(names[name], ownerOf(component)) { + if !lo.Contains(names[name], ownerOf(component)) { names[name] = append(names[name], ownerOf(component)) } } diff --git a/pkg/sbom/shippedscope/match_test.go b/pkg/sbom/shippedscope/match_test.go index 09d2fbd6f5..3d6515f269 100644 --- a/pkg/sbom/shippedscope/match_test.go +++ b/pkg/sbom/shippedscope/match_test.go @@ -14,7 +14,7 @@ var _ = Describe("Match, binary-backed components", func() { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "gopkg.in/yaml.v3"}, {Ecosystem: EcosystemGo, Name: "github.com/spf13/cobra"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{goBinaryNaming("gopkg.in/yaml.v3")}}, emptyIndex()) + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{goBinaryNaming("gopkg.in/yaml.v3")}}, emptyIndex()) Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNotLinked})) @@ -24,7 +24,7 @@ var _ = Describe("Match, binary-backed components", func() { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, {Ecosystem: EcosystemGo, Name: "b"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{goBinaryNaming("a", "b")}}, emptyIndex()) + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{goBinaryNaming("a", "b")}}, emptyIndex()) Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonBinary}))) }) @@ -32,7 +32,7 @@ var _ = Describe("Match, binary-backed components", func() { It("holds every module when any produced binary could carry modules but names none", func(ctx SpecContext) { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{ + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ goBinaryNaming("b"), {Go: VerdictUnknown, Rust: VerdictNone}, }}, emptyIndex()) @@ -40,10 +40,27 @@ var _ = Describe("Match, binary-backed components", func() { Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonFailOpen})) }) + It("reports a module a binary names as held by that binary, even when another binary is unreadable", func(ctx SpecContext) { + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemGo, Name: "a"}, + {Ecosystem: EcosystemGo, Name: "b"}, + {Ecosystem: EcosystemRust, Name: "serde", Version: "1"}, + {Ecosystem: EcosystemRust, Name: "tokio", Version: "1"}, + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ + {Go: VerdictNames, Modules: []string{"a"}, Rust: VerdictNames, Crates: []Crate{{Name: "serde", Version: "1"}}}, + {Go: VerdictUnknown, Rust: VerdictUnknown}, + }}, emptyIndex()) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) + Expect(decisions[1]).To(Equal(Decision{Shipped: true, Reason: ReasonFailOpen})) + Expect(decisions[2]).To(Equal(Decision{Shipped: true, Reason: ReasonBinary})) + Expect(decisions[3]).To(Equal(Decision{Shipped: true, Reason: ReasonFailOpen})) + }) + It("lets a test binary speak for nobody, whatever it names", func(ctx SpecContext) { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "github.com/stretchr/testify"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{ + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ {TestBinary: true, Go: VerdictNames, Modules: []string{"github.com/stretchr/testify"}}, }}, emptyIndex()) @@ -53,7 +70,7 @@ var _ = Describe("Match, binary-backed components", func() { It("does not let a test binary that cannot be read fail everything open", func(ctx SpecContext) { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{ + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ {TestBinary: true, Go: VerdictUnknown, Rust: VerdictUnknown}, }}, emptyIndex()) @@ -63,7 +80,7 @@ var _ = Describe("Match, binary-backed components", func() { It("drops every module when the build is known to have produced no binary", func(ctx SpecContext) { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, - }, &ProducedBinaries{}, emptyIndex()) + }, ProducedBinaries{Known: true}, emptyIndex()) Expect(decisions[0]).To(Equal(Decision{Reason: ReasonNotLinked})) }) @@ -72,7 +89,7 @@ var _ = Describe("Match, binary-backed components", func() { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.210"}, - }, nil, emptyIndex()) + }, ProducedBinaries{}, emptyIndex()) Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonNoData}))) }) @@ -82,7 +99,7 @@ var _ = Describe("Match, binary-backed components", func() { {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.210"}, {Ecosystem: EcosystemRust, Name: "serde", Version: "1.0.100"}, {Ecosystem: EcosystemGo, Name: "a", Version: "v9.9.9"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{ + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ {Go: VerdictNames, Modules: []string{"a"}, Rust: VerdictNames, Crates: []Crate{{Name: "serde", Version: "1.0.210"}}}, }}, emptyIndex()) @@ -95,7 +112,7 @@ var _ = Describe("Match, binary-backed components", func() { decisions := Match(ctx, []Component{ {Ecosystem: EcosystemGo, Name: "a"}, {Ecosystem: EcosystemRust, Name: "serde", Version: "1"}, - }, &ProducedBinaries{Binaries: []BinaryClassification{ + }, ProducedBinaries{Known: true, Binaries: []BinaryClassification{ {Go: VerdictNames, Modules: []string{"b"}, Rust: VerdictUnknown}, }}, emptyIndex()) @@ -105,7 +122,7 @@ var _ = Describe("Match, binary-backed components", func() { }) var _ = Describe("Match, file-backed components", func() { - noBinaries := &ProducedBinaries{} + noBinaries := ProducedBinaries{Known: true} It("holds a component whose own file is in the image byte for byte", func(ctx SpecContext) { index := indexOf(map[string]File{"/bin/jq": {Digest: "sha256:jq", Size: 10}}) @@ -156,10 +173,104 @@ var _ = Describe("Match, file-backed components", func() { }}, }, noBinaries, index) - Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 2, Matched: []string{"/usr/lib/libssl.so.3"}})) + // The proven owner is credited with the shared file as well: the rule decided it is + // its, and the next image in the chain may import only that file. + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 2, Matched: []string{"/usr/bin/c_rehash", "/usr/lib/libssl.so.3"}})) Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 2})) }) + It("treats two files with the same bytes at two paths as two files, each its owner's", func(ctx SpecContext) { + index := indexOf(map[string]File{ + "/x/f": {Digest: "sha256:d", Size: 10}, + "/y/f": {Digest: "sha256:d", Size: 10}, + "/y/g": {Digest: "sha256:g", Size: 10}, + }) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemPython, Name: "x", Version: "1", Occurrences: []Occurrence{{Path: "/x/f", Digest: "sha256:d"}}}, + {Ecosystem: EcosystemPython, Name: "y", Version: "1", Occurrences: []Occurrence{{Path: "/y/f", Digest: "sha256:d"}, {Path: "/y/g", Digest: "sha256:g"}}}, + }, noBinaries, index) + + Expect(decisions[0]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 1, Matched: []string{"/x/f"}})) + Expect(decisions[1]).To(Equal(Decision{Shipped: true, Reason: ReasonContent, Checked: 2, Matched: []string{"/y/f", "/y/g"}})) + }) + + It("lets every owner of the bytes contest a file nobody declares at its image path", func(ctx SpecContext) { + index := indexOf(map[string]File{"/bin/jq": {Digest: "sha256:jq", Size: 10}}) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "jq", Version: "1.6", Occurrences: []Occurrence{{Path: "/usr/bin/jq", Digest: "sha256:jq"}}}, + {Ecosystem: EcosystemOSPM, Name: "jq-static", Version: "1.6", Occurrences: []Occurrence{{Path: "/opt/jq/jq", Digest: "sha256:jq"}}}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonContentShared, Checked: 1, Matched: []string{"/bin/jq"}}))) + }) + + It("credits a content-shared component with every shared file it has in the image", func(ctx SpecContext) { + index := indexOf(map[string]File{ + "/s/a": {Digest: "sha256:a", Size: 10}, + "/s/b": {Digest: "sha256:b", Size: 10}, + }) + both := []Occurrence{{Path: "/s/a", Digest: "sha256:a"}, {Path: "/s/b", Digest: "sha256:b"}} + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "foo", Version: "1", Occurrences: both}, + {Ecosystem: EcosystemOSPM, Name: "foo-data", Version: "1", Occurrences: both}, + }, noBinaries, index) + + Expect(decisions).To(HaveEach(Equal(Decision{Shipped: true, Reason: ReasonContentShared, Checked: 2, Matched: []string{"/s/a", "/s/b"}}))) + }) + + It("drops every unproven owner of a shared file once one owner is proven, however many there are", func(ctx SpecContext) { + index := indexOf(map[string]File{ + "/usr/bin/c_rehash": {Digest: "sha256:rehash", Size: 10}, + "/usr/lib/libssl.so.3": {Digest: "sha256:ssl-13", Size: 10}, + }) + rehash := Occurrence{Path: "/usr/bin/c_rehash", Digest: "sha256:rehash"} + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemOSPM, Name: "openssl", Version: "3.0.11", Occurrences: []Occurrence{rehash, {Path: "/usr/lib/libssl.so.3", Digest: "sha256:ssl-11"}}}, + {Ecosystem: EcosystemOSPM, Name: "openssl", Version: "3.0.12", Occurrences: []Occurrence{rehash, {Path: "/usr/lib/libssl.so.3", Digest: "sha256:ssl-12"}}}, + {Ecosystem: EcosystemOSPM, Name: "openssl", Version: "3.0.13", Occurrences: []Occurrence{rehash, {Path: "/usr/lib/libssl.so.3", Digest: "sha256:ssl-13"}}}, + }, noBinaries, index) + + Expect(decisions[0].Shipped).To(BeFalse()) + Expect(decisions[1].Shipped).To(BeFalse()) + Expect(decisions[2].Reason).To(Equal(ReasonContent)) + }) + + It("does not let a copy of a package count as a proven co-owner against itself", func(ctx SpecContext) { + // Copy 1 of debug is proven by a file of its own. Copy 2 and `other` both declare + // /shared/f. Copy 2 is the same owner as copy 1, so it must not read copy 1's + // proof as a co-owner's and drop itself — while `other`, sharing the file with + // that proven owner, does drop. + index := indexOf(map[string]File{ + "/a/node_modules/debug/index.js": {Digest: "sha256:debug", Size: 10}, + "/shared/f": {Digest: "sha256:f", Size: 10}, + }) + + decisions := Match(ctx, []Component{ + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "2.6.9", Occurrences: []Occurrence{{Path: "/a/node_modules/debug/index.js", Digest: "sha256:debug"}}}, + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "2.6.9", Occurrences: []Occurrence{{Path: "/shared/f", Digest: "sha256:f"}}}, + {Ecosystem: EcosystemJavaScript, Name: "other", Version: "1", Occurrences: []Occurrence{{Path: "/shared/f", Digest: "sha256:f"}}}, + }, noBinaries, index) + + Expect(decisions[0].Reason).To(Equal(ReasonContent)) + Expect(decisions[1].Reason).To(Equal(ReasonContentShared)) + Expect(decisions[2].Reason).To(Equal(ReasonNoMatch)) + }) + + It("holds a component made only of symlinks by their paths, and drops it when they are gone", func(ctx SpecContext) { + present := &FileIndex{Files: map[string]File{}, Symlinks: map[string]string{"/usr/bin/python": "python3"}} + absent := emptyIndex() + component := []Component{ + {Ecosystem: EcosystemOSPM, Name: "python-is-python3", Version: "3", Occurrences: []Occurrence{{Path: "/usr/bin/python", Symlink: true}}}, + } + + Expect(Match(ctx, component, noBinaries, present)[0]).To(Equal(Decision{Shipped: true, Reason: ReasonPath, Matched: []string{"/usr/bin/python"}})) + Expect(Match(ctx, component, noBinaries, absent)[0]).To(Equal(Decision{Reason: ReasonNoMatch})) + }) + It("holds both owners of a shared file when neither proved itself otherwise", func(ctx SpecContext) { index := indexOf(map[string]File{"/usr/share/x/data": {Digest: "sha256:data", Size: 10}}) @@ -233,7 +344,7 @@ var _ = Describe("Match, file-backed components", func() { }, noBinaries, index) Expect(decisions[0].Reason).To(Equal(ReasonContent)) - Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 2})) + Expect(decisions[1]).To(Equal(Decision{Reason: ReasonNoMatch, Checked: 1})) }) It("holds a relocated and rewritten file by its unique name", func(ctx SpecContext) { @@ -302,7 +413,7 @@ var _ = Describe("Match, the concept's end-to-end stand", func() { "/lib/x86_64/libc.so.6": {Digest: "sha256:libc", Size: 10}, }) - produced := &ProducedBinaries{Binaries: []BinaryClassification{ + produced := ProducedBinaries{Known: true, Binaries: []BinaryClassification{ {Go: VerdictNames, Modules: []string{"gopkg.in/yaml.v3"}, Rust: VerdictNone}, }} @@ -328,28 +439,28 @@ var _ = Describe("Match, the concept's end-to-end stand", func() { decisions := Match(ctx, components, produced, final) - shipped := map[string]bool{} + reasons := map[string]Reason{} for i, component := range components { - shipped[string(component.Ecosystem)+":"+component.Name] = decisions[i].Shipped + reasons[string(component.Ecosystem)+":"+component.Name] = decisions[i].Reason } - Expect(shipped).To(Equal(map[string]bool{ - "go:gopkg.in/yaml.v3": true, - "go:github.com/spf13/cobra": false, - "go:github.com/spf13/pflag": false, - "python:requests": true, - "python:certifi": false, - "python:idna": false, - "python:pytest": false, - "javascript:lodash": true, - "javascript:jest": false, - "javascript:@babel/cli": false, - "os-pm:jq": true, - "os-pm:libpcre3": false, - "os-pm:zlib1g": true, - "os-pm:libc6": true, - "os-pm:gcc-12": false, - "os-pm:python3-pip": false, - "os-pm:npm": false, + Expect(reasons).To(Equal(map[string]Reason{ + "go:gopkg.in/yaml.v3": ReasonBinary, + "go:github.com/spf13/cobra": ReasonNotLinked, + "go:github.com/spf13/pflag": ReasonNotLinked, + "python:requests": ReasonContent, + "python:certifi": ReasonNoMatch, + "python:idna": ReasonNoMatch, + "python:pytest": ReasonNoMatch, + "javascript:lodash": ReasonContent, + "javascript:jest": ReasonNoMatch, + "javascript:@babel/cli": ReasonNoMatch, + "os-pm:jq": ReasonContent, + "os-pm:libpcre3": ReasonNoMatch, + "os-pm:zlib1g": ReasonContent, + "os-pm:libc6": ReasonContent, + "os-pm:gcc-12": ReasonNoMatch, + "os-pm:python3-pip": ReasonNoMatch, + "os-pm:npm": ReasonNoMatch, })) }) })