feat(sbom): mark the components whose files an image does not carry

pkg/build/sbom_step.go

@@ -207,7 +207,7 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string,
 
 		cyclonedxutil.Canonicalize(ctx, resultBOM)
 
-		recordShippedFiles(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name)
+		recordShippedScope(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name)
 
 		resultJSON, err := cyclonedxutil.ToJSON(resultBOM)
 		if err != nil {
@@ -228,10 +228,11 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string,
 	})
 }
 
-// recordShippedFiles records, on every component an ecosystem installed into the image,
-// which files of the image are that component's. An image built from this one can then
-// ask of an inherited component whether its files are still there, instead of trusting
-// that a component its base declared came along.
+// recordShippedScope records, on every component an ecosystem installed into the image,
+// which files of the image are that component's, and then marks the components whose
+// files are not in this image at all. An image built from this one can then ask of an
+// inherited component whether its files are still there, instead of trusting that a
+// component its base declared came along.
 //
 // Runs last, once the BOM holds the components it will ship with: Canonicalize folds
 // duplicate components together and drops an evidence list that comes out empty, which is
@@ -241,11 +242,11 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string,
 // second time: the read is then as long as a save to local disk, and what can fail it is
 // the host — an image that is not there, a disk that is full — rather than the network.
 //
-// Nothing here fails the build. A BOM without these records is the BOM werf wrote before
-// it kept them, and a reader that finds no record of a component's files keeps the
-// component, so the worst an unreadable image costs is the SBOM of the image built from
-// this one being as complete as it used to be.
-func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) {
+// Nothing here fails the build, and nothing is removed from the BOM. A BOM without these
+// records is the BOM werf wrote before it kept them, and a reader that finds no record of
+// a component's files keeps the component, so the worst an unreadable image costs is the
+// SBOM of the image built from this one being as complete as it used to be.
+func recordShippedScope(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) {
 	if !shippedscope.HasFileBackedComponents(bom) {
 		return
 	}
@@ -267,6 +268,10 @@ func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, w
 
 	recorded := shippedscope.RecordOccurrences(bom, filesets.Filesets)
 	logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded)
+
+	if marked := shippedscope.MarkNotShipped(ctx, bom, filesets); marked > 0 {
+		logboek.Context(ctx).Default().LogF("image %s: %d components of the SBOM are not shipped by the image\n", werfImgName, marked)
+	}
 }
 
 // savedImage reads the image out of the backend as a docker-save tarball. The tarball is
@@ -548,7 +553,7 @@ func (step *sbomStep) recordDeclaredPackages(ctx context.Context, bom *cdx.BOM,
 	return nil
 }
 
-const sbomArtifactFormatVersion = "11"
+const sbomArtifactFormatVersion = "12"
 
 // calculateStableChecksum computes the SBOM artifact cache checksum. Together with the
 // parent stage digest it forms the cache key: a previously attached SBOM is reused only

pkg/build/sbom_step_shipped_files_test.go

@@ -44,7 +44,7 @@ func (b *savedImageBackend) SaveImageToStream(_ context.Context, imageName strin
 	return io.NopCloser(bytes.NewReader(b.tarball)), nil
 }
 
-var _ = Describe("recordShippedFiles", func() {
+var _ = Describe("recordShippedScope", func() {
 	tarOf := func(files, symlinks map[string]string) []byte {
 		var buf bytes.Buffer
 		writer := tar.NewWriter(&buf)
@@ -146,7 +146,7 @@ var _ = Describe("recordShippedFiles", func() {
 			{BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"},
 		}}
 
-		recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+		recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
 
 		Expect(backend.saved).To(Equal([]string{"werf-stages:app"}))
 		components := *bom.Components
@@ -171,12 +171,107 @@ var _ = Describe("recordShippedFiles", func() {
 		}, nil)
 
 		bom := &cdx.BOM{Components: &[]cdx.Component{
-			{BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"},
+			{BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{
+				{Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("a body this image does not hold")},
+			}}},
 		}}
 
-		recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+		out := logged(ctx, func(ctx context.Context) {
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+		})
+
+		Expect(out).To(ContainSubstring("/var/lib/pm/files.json is not"))
+		Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty())
+		Expect(lo.FromPtr((*bom.Components)[0].Evidence.Occurrences)).To(HaveLen(1),
+			"what the donor recorded is left as it was")
+	})
 
-		Expect((*bom.Components)[0].Evidence).To(BeNil())
+	It("keeps a component whose files this image moved away from the paths its manifest declares", func(ctx SpecContext) {
+		backend := backendWith(map[string]string{
+			"/usr/lib/python3/moved-1.0.0.dist-info/RECORD": "moved/__init__.py,sha256=x,12\n",
+			"/opt/relocated/moved/__init__.py":              "moved body",
+		}, nil)
+
+		bom := &cdx.BOM{Components: &[]cdx.Component{
+			{BOMRef: "moved", Name: "moved", PackageURL: "pkg:pypi/moved@1.0.0", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{
+				{Location: "/usr/lib/python3/moved/__init__.py", AdditionalContext: digestOf("moved body")},
+			}}},
+		}}
+
+		recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+
+		Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty(),
+			"the bytes are in the image, so the manifest naming a path they left must not mark the component")
+	})
+
+	It("marks a component the image installed and then deleted, leaving only its manifest", func(ctx SpecContext) {
+		backend := backendWith(map[string]string{
+			"/usr/lib/python3/tool-1.0.0.dist-info/RECORD": "tool/main.py,sha256=x,12\n",
+		}, nil)
+
+		bom := &cdx.BOM{Components: &[]cdx.Component{
+			{BOMRef: "tool", Name: "tool", PackageURL: "pkg:pypi/tool@1.0.0"},
+		}}
+
+		recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+
+		Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(ContainElement(cdx.Property{Name: "werf:shipped", Value: "false"}),
+			"a build toolchain removed by a later stage is not shipped, and its own manifest is all that could say so")
+	})
+
+	It("clears a verdict a donor recorded on a component this image does ship", func(ctx SpecContext) {
+		backend := backendWith(map[string]string{"/bin/jq": "jq body"}, nil)
+
+		bom := &cdx.BOM{Components: &[]cdx.Component{
+			{
+				BOMRef: "jq", Name: "jq", PackageURL: "pkg:generic/jq@1.8.1",
+				Properties: &[]cdx.Property{{Name: "werf:shipped", Value: "false"}},
+				Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{
+					{Location: "/usr/bin/jq", AdditionalContext: digestOf("jq body")},
+				}},
+			},
+		}}
+
+		recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+
+		Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty())
+	})
+
+	It("marks a component the image's own file lists do not account for and whose files it does not hold", func(ctx SpecContext) {
+		backend := backendWith(map[string]string{
+			"/usr/lib/python3/kept-1.0.0.dist-info/RECORD": "kept/__init__.py,sha256=x,12\n",
+			"/usr/lib/python3/kept/__init__.py":            "kept body",
+		}, nil)
+
+		bom := &cdx.BOM{Components: &[]cdx.Component{
+			{BOMRef: "kept", Name: "kept", PackageURL: "pkg:pypi/kept@1.0.0"},
+			{BOMRef: "gone", Name: "gone", PackageURL: "pkg:pypi/gone@2.0.0", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{
+				{Location: "/usr/lib/python3/gone/__init__.py", AdditionalContext: digestOf("gone body")},
+			}}},
+		}}
+
+		out := logged(ctx, func(ctx context.Context) {
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+		})
+
+		components := *bom.Components
+		Expect(lo.FromPtr(components[0].Properties)).To(BeEmpty(), "the image holds the files this component declares")
+		Expect(lo.FromPtr(components[1].Properties)).To(ContainElement(cdx.Property{Name: "werf:shipped", Value: "false"}))
+		Expect(out).To(ContainSubstring("image app: 1 components of the SBOM are not shipped by the image"))
+	})
+
+	It("keeps a component whose files an import put somewhere else in the image", func(ctx SpecContext) {
+		backend := backendWith(map[string]string{"/bin/jq": "jq body"}, nil)
+
+		bom := &cdx.BOM{Components: &[]cdx.Component{
+			{BOMRef: "jq", Name: "jq", PackageURL: "pkg:generic/jq@1.8.1", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{
+				{Location: "/usr/bin/jq", AdditionalContext: digestOf("jq body")},
+			}}},
+		}}
+
+		recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
+
+		Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty())
 	})
 
 	It("warns and leaves the BOM as it was when the backend cannot save the image", func(ctx SpecContext) {
@@ -186,7 +281,7 @@ var _ = Describe("recordShippedFiles", func() {
 		}}
 
 		out := logged(ctx, func(ctx context.Context) {
-			recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
 		})
 
 		Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app"))
@@ -201,7 +296,7 @@ var _ = Describe("recordShippedFiles", func() {
 		}}
 
 		out := logged(ctx, func(ctx context.Context) {
-			recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
 		})
 
 		Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app"))
@@ -220,7 +315,7 @@ var _ = Describe("recordShippedFiles", func() {
 		}}
 
 		out := logged(ctx, func(ctx context.Context) {
-			recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
 		})
 
 		Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image werf-stages:app consist of"))
@@ -236,7 +331,7 @@ var _ = Describe("recordShippedFiles", func() {
 		}}
 
 		out := logged(ctx, func(ctx context.Context) {
-			recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app")
+			recordShippedScope(ctx, bom, backend, "app", "werf-stages:app")
 		})
 
 		Expect(out).To(BeEmpty())

pkg/sbom/shippedscope/fileset.go

@@ -37,14 +37,21 @@ type Fileset struct {
 
 type Filesets struct {
 	Filesets []Fileset
+	// Index is what the image holds, the same read the filesets were drawn from. A consumer
+	// that has to look up a file the filesets do not name — a component's file under a path
+	// no manifest of this image declares — takes it from here rather than reading the image
+	// a second time.
+	Index *FileIndex
 	// OSPMFileListMissing is set when the image carries os-pm packages but was installed
 	// by a pm that recorded no file list for them, so none of them has a fileset here. A
 	// consumer that would drop a component without a fileset must not do so for os-pm then.
 	OSPMFileListMissing bool
 }
 
 // CollectFilesets reads the image once and answers, for every component an ecosystem
-// installed into it, which files are that component's.
+// installed into it, which files are that component's. The index that read produced comes
+// back with the filesets, never nil, so that a consumer needing a file no fileset names
+// does not read the image again.
 //
 // Each ecosystem draws the boundary itself, and none of them draw it around a directory:
 // Python takes the paths its RECORD declares, JavaScript takes a package directory minus
@@ -67,6 +74,7 @@ func CollectFilesets(ctx context.Context, img v1.Image) (*Filesets, error) {
 
 	filesets := &Filesets{
 		Filesets:            osPM,
+		Index:               index,
 		OSPMFileListMissing: missing,
 	}
 	filesets.Filesets = append(filesets.Filesets, pythonFilesets(ctx, index, manifests)...)

pkg/sbom/shippedscope/mark.go

@@ -0,0 +1,118 @@
+package shippedscope
+
+import (
+	"context"
+
+	cdx "github.com/CycloneDX/cyclonedx-go"
+	"github.com/samber/lo"
+
+	"github.com/werf/logboek"
+)
+
+// PropertyShipped is written on a component the image does not ship. Only the negative
+// case is recorded: a component without the property is one the image ships, one nothing
+// is known about, or one written by a werf that did not look.
+//
+// The property describes one image, so it is the verdict of the image whose SBOM carries
+// it and of no other. A component inherited from a donor arrives carrying the donor's
+// verdict, which says where the component was in the donor's image rather than in this
+// one, and MarkNotShipped drops it before deciding anew.
+const PropertyShipped = "werf:shipped"
+
+const propertyShippedFalse = "false"
+
+// MarkNotShipped decides, for every component of the BOM whose files an ecosystem can
+// describe, whether this image ships it, and writes PropertyShipped on the ones it does
+// not. Nothing is removed: a component the rule calls absent is still a component the
+// image's SBOM declared, and this first increment records the verdict to be looked at on
+// real builds before anything acts on it.
+//
+// Runs after RecordOccurrences, on the occurrences it leaves: for a component this image
+// installed those are the files the image holds, and for one inherited from a donor whose
+// files this image has no manifest for they are the donor's, which is what lets the rule
+// find them by content wherever an import put them.
+//
+// Go modules and crates are kept untouched. Each is proven by the binaries this image's
+// own build produced, and that set is not established here, so the rule is given none and
+// fails open on all of them.
+//
+// Components are passed to the rule together rather than one at a time: whether a file
+// proves the component that claims it depends on which other components claim the same
+// bytes.
+func MarkNotShipped(ctx context.Context, bom *cdx.BOM, filesets *Filesets) int {
+	if bom == nil || filesets == nil || filesets.Index == nil {
+		return 0
+	}
+
+	var components []Component
+	var judged []*cdx.Component
+
+	walkComponents(bom, func(comp *cdx.Component) {
+		removeProperty(comp, PropertyShipped)
+
+		key, ok := keyOfComponent(comp)
+		if !ok {
+			return
+		}
+		// The image carries os-pm packages installed by a pm that wrote no file list, so
+		// what the rule would weigh for them is a donor's record of files this image
+		// cannot confirm either way.
+		if key.ecosystem == EcosystemOSPM && filesets.OSPMFileListMissing {
+			return
+		}
+
+		components = append(components, Component{
+			Ecosystem:   key.ecosystem,
+			Name:        key.name,
+			Version:     key.version,
+			Occurrences: recordedOccurrences(comp),
+		})
+		judged = append(judged, comp)
+	})
+
+	if len(components) == 0 {
+		return 0
+	}
+
+	count := 0
+	for i, decision := range Match(ctx, components, ProducedBinaries{}, filesets.Index) {
+		if decision.Shipped {
+			continue
+		}
+
+		setProperty(judged[i], PropertyShipped, propertyShippedFalse)
+		count++
+
+		logboek.Context(ctx).Debug().LogF("not shipped %s %s@%s (%s): checked %d files, 0 matches\n",
+			components[i].Ecosystem, components[i].Name, components[i].Version, decision.Reason, decision.Checked)
+	}
+
+	return count
+}
+
+func setProperty(comp *cdx.Component, name, value string) {
+	properties := lo.FromPtr(comp.Properties)
+	for i := range properties {
+		if properties[i].Name == name {
+			properties[i].Value = value
+			comp.Properties = &properties
+
+			return
+		}
+	}
+
+	properties = append(properties, cdx.Property{Name: name, Value: value})
+	comp.Properties = &properties
+}
+
+func removeProperty(comp *cdx.Component, name string) {
+	properties := lo.FromPtr(comp.Properties)
+	kept := lo.Filter(properties, func(property cdx.Property, _ int) bool {
+		return property.Name != name
+	})
+	if len(kept) == len(properties) {
+		return
+	}
+
+	comp.Properties = &kept
+}