diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index c73a706f0a..c4a4299ad4 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -207,7 +207,7 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) - recordShippedFiles(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name) + recordShippedScope(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name) resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { @@ -228,10 +228,11 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, }) } -// recordShippedFiles records, on every component an ecosystem installed into the image, -// which files of the image are that component's. An image built from this one can then -// ask of an inherited component whether its files are still there, instead of trusting -// that a component its base declared came along. +// recordShippedScope records, on every component an ecosystem installed into the image, +// which files of the image are that component's, and then marks the components whose +// files are not in this image at all. An image built from this one can then ask of an +// inherited component whether its files are still there, instead of trusting that a +// component its base declared came along. // // Runs last, once the BOM holds the components it will ship with: Canonicalize folds // duplicate components together and drops an evidence list that comes out empty, which is @@ -241,11 +242,11 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, // second time: the read is then as long as a save to local disk, and what can fail it is // the host — an image that is not there, a disk that is full — rather than the network. // -// Nothing here fails the build. A BOM without these records is the BOM werf wrote before -// it kept them, and a reader that finds no record of a component's files keeps the -// component, so the worst an unreadable image costs is the SBOM of the image built from -// this one being as complete as it used to be. -func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) { +// Nothing here fails the build, and nothing is removed from the BOM. A BOM without these +// records is the BOM werf wrote before it kept them, and a reader that finds no record of +// a component's files keeps the component, so the worst an unreadable image costs is the +// SBOM of the image built from this one being as complete as it used to be. +func recordShippedScope(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) { if !shippedscope.HasFileBackedComponents(bom) { return } @@ -267,6 +268,10 @@ func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, w recorded := shippedscope.RecordOccurrences(bom, filesets.Filesets) logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) + + if marked := shippedscope.MarkNotShipped(ctx, bom, filesets); marked > 0 { + logboek.Context(ctx).Default().LogF("image %s: %d components of the SBOM are not shipped by the image\n", werfImgName, marked) + } } // savedImage reads the image out of the backend as a docker-save tarball. The tarball is @@ -548,7 +553,7 @@ func (step *sbomStep) recordDeclaredPackages(ctx context.Context, bom *cdx.BOM, return nil } -const sbomArtifactFormatVersion = "11" +const sbomArtifactFormatVersion = "12" // calculateStableChecksum computes the SBOM artifact cache checksum. Together with the // parent stage digest it forms the cache key: a previously attached SBOM is reused only diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go index c92b9d3b82..b6e3ba29d7 100644 --- a/pkg/build/sbom_step_shipped_files_test.go +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -44,7 +44,7 @@ func (b *savedImageBackend) SaveImageToStream(_ context.Context, imageName strin return io.NopCloser(bytes.NewReader(b.tarball)), nil } -var _ = Describe("recordShippedFiles", func() { +var _ = Describe("recordShippedScope", func() { tarOf := func(files, symlinks map[string]string) []byte { var buf bytes.Buffer writer := tar.NewWriter(&buf) @@ -146,7 +146,7 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") Expect(backend.saved).To(Equal([]string{"werf-stages:app"})) components := *bom.Components @@ -171,12 +171,107 @@ var _ = Describe("recordShippedFiles", func() { }, nil) bom := &cdx.BOM{Components: &[]cdx.Component{ - {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("a body this image does not hold")}, + }}}, }} - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + out := logged(ctx, func(ctx context.Context) { + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + }) + + Expect(out).To(ContainSubstring("/var/lib/pm/files.json is not")) + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty()) + Expect(lo.FromPtr((*bom.Components)[0].Evidence.Occurrences)).To(HaveLen(1), + "what the donor recorded is left as it was") + }) - Expect((*bom.Components)[0].Evidence).To(BeNil()) + It("keeps a component whose files this image moved away from the paths its manifest declares", func(ctx SpecContext) { + backend := backendWith(map[string]string{ + "/usr/lib/python3/moved-1.0.0.dist-info/RECORD": "moved/__init__.py,sha256=x,12\n", + "/opt/relocated/moved/__init__.py": "moved body", + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "moved", Name: "moved", PackageURL: "pkg:pypi/moved@1.0.0", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/moved/__init__.py", AdditionalContext: digestOf("moved body")}, + }}}, + }} + + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty(), + "the bytes are in the image, so the manifest naming a path they left must not mark the component") + }) + + It("marks a component the image installed and then deleted, leaving only its manifest", func(ctx SpecContext) { + backend := backendWith(map[string]string{ + "/usr/lib/python3/tool-1.0.0.dist-info/RECORD": "tool/main.py,sha256=x,12\n", + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "tool", Name: "tool", PackageURL: "pkg:pypi/tool@1.0.0"}, + }} + + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(ContainElement(cdx.Property{Name: "werf:shipped", Value: "false"}), + "a build toolchain removed by a later stage is not shipped, and its own manifest is all that could say so") + }) + + It("clears a verdict a donor recorded on a component this image does ship", func(ctx SpecContext) { + backend := backendWith(map[string]string{"/bin/jq": "jq body"}, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + { + BOMRef: "jq", Name: "jq", PackageURL: "pkg:generic/jq@1.8.1", + Properties: &[]cdx.Property{{Name: "werf:shipped", Value: "false"}}, + Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/bin/jq", AdditionalContext: digestOf("jq body")}, + }}, + }, + }} + + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty()) + }) + + It("marks a component the image's own file lists do not account for and whose files it does not hold", func(ctx SpecContext) { + backend := backendWith(map[string]string{ + "/usr/lib/python3/kept-1.0.0.dist-info/RECORD": "kept/__init__.py,sha256=x,12\n", + "/usr/lib/python3/kept/__init__.py": "kept body", + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "kept", Name: "kept", PackageURL: "pkg:pypi/kept@1.0.0"}, + {BOMRef: "gone", Name: "gone", PackageURL: "pkg:pypi/gone@2.0.0", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/gone/__init__.py", AdditionalContext: digestOf("gone body")}, + }}}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + }) + + components := *bom.Components + Expect(lo.FromPtr(components[0].Properties)).To(BeEmpty(), "the image holds the files this component declares") + Expect(lo.FromPtr(components[1].Properties)).To(ContainElement(cdx.Property{Name: "werf:shipped", Value: "false"})) + Expect(out).To(ContainSubstring("image app: 1 components of the SBOM are not shipped by the image")) + }) + + It("keeps a component whose files an import put somewhere else in the image", func(ctx SpecContext) { + backend := backendWith(map[string]string{"/bin/jq": "jq body"}, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "jq", Name: "jq", PackageURL: "pkg:generic/jq@1.8.1", Evidence: &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/bin/jq", AdditionalContext: digestOf("jq body")}, + }}}, + }} + + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") + + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(BeEmpty()) }) It("warns and leaves the BOM as it was when the backend cannot save the image", func(ctx SpecContext) { @@ -186,7 +281,7 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") }) Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) @@ -201,7 +296,7 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") }) Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) @@ -220,7 +315,7 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") }) Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image werf-stages:app consist of")) @@ -236,7 +331,7 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + recordShippedScope(ctx, bom, backend, "app", "werf-stages:app") }) Expect(out).To(BeEmpty()) diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go index 1ba1efaab0..af562cdb31 100644 --- a/pkg/sbom/shippedscope/fileset.go +++ b/pkg/sbom/shippedscope/fileset.go @@ -37,6 +37,11 @@ type Fileset struct { type Filesets struct { Filesets []Fileset + // Index is what the image holds, the same read the filesets were drawn from. A consumer + // that has to look up a file the filesets do not name — a component's file under a path + // no manifest of this image declares — takes it from here rather than reading the image + // a second time. + Index *FileIndex // OSPMFileListMissing is set when the image carries os-pm packages but was installed // by a pm that recorded no file list for them, so none of them has a fileset here. A // consumer that would drop a component without a fileset must not do so for os-pm then. @@ -44,7 +49,9 @@ type Filesets struct { } // CollectFilesets reads the image once and answers, for every component an ecosystem -// installed into it, which files are that component's. +// installed into it, which files are that component's. The index that read produced comes +// back with the filesets, never nil, so that a consumer needing a file no fileset names +// does not read the image again. // // Each ecosystem draws the boundary itself, and none of them draw it around a directory: // Python takes the paths its RECORD declares, JavaScript takes a package directory minus @@ -67,6 +74,7 @@ func CollectFilesets(ctx context.Context, img v1.Image) (*Filesets, error) { filesets := &Filesets{ Filesets: osPM, + Index: index, OSPMFileListMissing: missing, } filesets.Filesets = append(filesets.Filesets, pythonFilesets(ctx, index, manifests)...) diff --git a/pkg/sbom/shippedscope/mark.go b/pkg/sbom/shippedscope/mark.go new file mode 100644 index 0000000000..fde38fd7b2 --- /dev/null +++ b/pkg/sbom/shippedscope/mark.go @@ -0,0 +1,118 @@ +package shippedscope + +import ( + "context" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/samber/lo" + + "github.com/werf/logboek" +) + +// PropertyShipped is written on a component the image does not ship. Only the negative +// case is recorded: a component without the property is one the image ships, one nothing +// is known about, or one written by a werf that did not look. +// +// The property describes one image, so it is the verdict of the image whose SBOM carries +// it and of no other. A component inherited from a donor arrives carrying the donor's +// verdict, which says where the component was in the donor's image rather than in this +// one, and MarkNotShipped drops it before deciding anew. +const PropertyShipped = "werf:shipped" + +const propertyShippedFalse = "false" + +// MarkNotShipped decides, for every component of the BOM whose files an ecosystem can +// describe, whether this image ships it, and writes PropertyShipped on the ones it does +// not. Nothing is removed: a component the rule calls absent is still a component the +// image's SBOM declared, and this first increment records the verdict to be looked at on +// real builds before anything acts on it. +// +// Runs after RecordOccurrences, on the occurrences it leaves: for a component this image +// installed those are the files the image holds, and for one inherited from a donor whose +// files this image has no manifest for they are the donor's, which is what lets the rule +// find them by content wherever an import put them. +// +// Go modules and crates are kept untouched. Each is proven by the binaries this image's +// own build produced, and that set is not established here, so the rule is given none and +// fails open on all of them. +// +// Components are passed to the rule together rather than one at a time: whether a file +// proves the component that claims it depends on which other components claim the same +// bytes. +func MarkNotShipped(ctx context.Context, bom *cdx.BOM, filesets *Filesets) int { + if bom == nil || filesets == nil || filesets.Index == nil { + return 0 + } + + var components []Component + var judged []*cdx.Component + + walkComponents(bom, func(comp *cdx.Component) { + removeProperty(comp, PropertyShipped) + + key, ok := keyOfComponent(comp) + if !ok { + return + } + // The image carries os-pm packages installed by a pm that wrote no file list, so + // what the rule would weigh for them is a donor's record of files this image + // cannot confirm either way. + if key.ecosystem == EcosystemOSPM && filesets.OSPMFileListMissing { + return + } + + components = append(components, Component{ + Ecosystem: key.ecosystem, + Name: key.name, + Version: key.version, + Occurrences: recordedOccurrences(comp), + }) + judged = append(judged, comp) + }) + + if len(components) == 0 { + return 0 + } + + count := 0 + for i, decision := range Match(ctx, components, ProducedBinaries{}, filesets.Index) { + if decision.Shipped { + continue + } + + setProperty(judged[i], PropertyShipped, propertyShippedFalse) + count++ + + logboek.Context(ctx).Debug().LogF("not shipped %s %s@%s (%s): checked %d files, 0 matches\n", + components[i].Ecosystem, components[i].Name, components[i].Version, decision.Reason, decision.Checked) + } + + return count +} + +func setProperty(comp *cdx.Component, name, value string) { + properties := lo.FromPtr(comp.Properties) + for i := range properties { + if properties[i].Name == name { + properties[i].Value = value + comp.Properties = &properties + + return + } + } + + properties = append(properties, cdx.Property{Name: name, Value: value}) + comp.Properties = &properties +} + +func removeProperty(comp *cdx.Component, name string) { + properties := lo.FromPtr(comp.Properties) + kept := lo.Filter(properties, func(property cdx.Property, _ int) bool { + return property.Name != name + }) + if len(kept) == len(properties) { + return + } + + comp.Properties = &kept +} diff --git a/pkg/sbom/shippedscope/mark_test.go b/pkg/sbom/shippedscope/mark_test.go new file mode 100644 index 0000000000..509bf1ac85 --- /dev/null +++ b/pkg/sbom/shippedscope/mark_test.go @@ -0,0 +1,236 @@ +package shippedscope + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" +) + +var _ = Describe("MarkNotShipped", func() { + componentOf := func(purl string, occurrences ...cdx.EvidenceOccurrence) cdx.Component { + comp := cdx.Component{BOMRef: purl, Name: purl, PackageURL: purl} + if occurrences != nil { + comp.Evidence = &cdx.Evidence{Occurrences: &occurrences} + } + + return comp + } + + bomOf := func(components ...cdx.Component) *cdx.BOM { + return &cdx.BOM{Components: &components} + } + + fileOccurrence := func(path, body string) cdx.EvidenceOccurrence { + return cdx.EvidenceOccurrence{Location: path, AdditionalContext: digestOf(body)} + } + + shippedProperty := func(bom *cdx.BOM, index int) (string, bool) { + for _, property := range lo.FromPtr((*bom.Components)[index].Properties) { + if property.Name == PropertyShipped { + return property.Value, true + } + } + + return "", false + } + + filesetsOf := func(index *FileIndex) *Filesets { + return &Filesets{Index: index} + } + + It("marks a component whose declared files are not in the image", func(ctx SpecContext) { + bom := bomOf( + componentOf("pkg:pypi/present@1.0.0", fileOccurrence("/app/present.py", "here")), + componentOf("pkg:pypi/gone@2.0.0", fileOccurrence("/app/gone.py", "elsewhere")), + ) + + marked := MarkNotShipped(ctx, bom, filesetsOf(indexOf(map[string]File{ + "/app/present.py": {Digest: digestOf("here"), Size: 4}, + }))) + + Expect(marked).To(Equal(1)) + _, present := shippedProperty(bom, 0) + Expect(present).To(BeFalse(), "a component the image ships carries no property") + value, found := shippedProperty(bom, 1) + Expect(found).To(BeTrue()) + Expect(value).To(Equal("false")) + }) + + It("marks a component whose files were looked for and none were found", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:pypi/gone@2.0.0")) + (*bom.Components)[0].Evidence = &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{}} + + marked := MarkNotShipped(ctx, bom, filesetsOf(emptyIndex())) + + Expect(marked).To(Equal(1)) + value, found := shippedProperty(bom, 0) + Expect(found).To(BeTrue()) + Expect(value).To(Equal("false")) + }) + + It("keeps a component nothing is known about", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:pypi/unknown@1.0.0")) + + marked := MarkNotShipped(ctx, bom, filesetsOf(emptyIndex())) + + Expect(marked).To(BeZero()) + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse()) + }) + + It("keeps a component whose only occurrences came from elsewhere", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:pypi/scanned@1.0.0", + cdx.EvidenceOccurrence{Location: "/app/scanned.py"}, + cdx.EvidenceOccurrence{Location: "/app/other.py", AdditionalContext: "seen by a scanner"}, + )) + + marked := MarkNotShipped(ctx, bom, filesetsOf(emptyIndex())) + + Expect(marked).To(BeZero()) + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse()) + }) + + It("finds a component by content wherever an import put it", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:generic/jq@1.8.1", fileOccurrence("/usr/bin/jq", "jq"))) + + marked := MarkNotShipped(ctx, bom, filesetsOf(indexOf(map[string]File{ + "/bin/jq": {Digest: digestOf("jq"), Size: 2}, + }))) + + Expect(marked).To(BeZero()) + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse()) + }) + + It("keeps every os-pm component when the image records no file lists", func(ctx SpecContext) { + bom := bomOf( + componentOf("pkg:generic/gone@1.0.0", fileOccurrence("/usr/lib/gone.so", "gone")), + componentOf("pkg:pypi/gone@2.0.0", fileOccurrence("/app/gone.py", "gone too")), + ) + + filesets := filesetsOf(emptyIndex()) + filesets.OSPMFileListMissing = true + + marked := MarkNotShipped(ctx, bom, filesets) + + Expect(marked).To(Equal(1), "only the component of an ecosystem the image does describe is judged") + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse()) + _, found = shippedProperty(bom, 1) + Expect(found).To(BeTrue()) + }) + + It("keeps a Go module, whose binaries this image's produced set does not cover", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:golang/github.com/spf13/cobra@1.8.0")) + + marked := MarkNotShipped(ctx, bom, filesetsOf(emptyIndex())) + + Expect(marked).To(BeZero()) + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse()) + }) + + It("marks a nested component", func(ctx SpecContext) { + nested := []cdx.Component{componentOf("pkg:pypi/gone@2.0.0", fileOccurrence("/app/gone.py", "gone"))} + outer := componentOf("pkg:pypi/outer@1.0.0", fileOccurrence("/app/outer.py", "outer")) + outer.Components = &nested + + bom := bomOf(outer) + + marked := MarkNotShipped(ctx, bom, filesetsOf(indexOf(map[string]File{ + "/app/outer.py": {Digest: digestOf("outer"), Size: 5}, + }))) + + Expect(marked).To(Equal(1)) + value, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse(), "the outer component ships") + Expect(value).To(BeEmpty()) + + for _, property := range lo.FromPtr((*(*bom.Components)[0].Components)[0].Properties) { + if property.Name == PropertyShipped { + Expect(property.Value).To(Equal("false")) + + return + } + } + Fail("the nested component carries no shipped property") + }) + + It("clears a verdict a donor recorded when this image does ship the component", func(ctx SpecContext) { + comp := componentOf("pkg:pypi/present@1.0.0", fileOccurrence("/app/present.py", "here")) + comp.Properties = &[]cdx.Property{{Name: PropertyShipped, Value: "false"}} + + bom := bomOf(comp) + + marked := MarkNotShipped(ctx, bom, filesetsOf(indexOf(map[string]File{ + "/app/present.py": {Digest: digestOf("here"), Size: 4}, + }))) + + Expect(marked).To(BeZero()) + _, found := shippedProperty(bom, 0) + Expect(found).To(BeFalse(), "the donor's verdict describes the donor's image, not this one") + }) + + It("clears a verdict a donor recorded on a component this image does not judge", func(ctx SpecContext) { + module := componentOf("pkg:golang/github.com/spf13/cobra@1.8.0") + module.Properties = &[]cdx.Property{{Name: PropertyShipped, Value: "false"}} + osPM := componentOf("pkg:generic/zlib@1.3.1", fileOccurrence("/usr/lib/libz.so", "z")) + osPM.Properties = &[]cdx.Property{{Name: PropertyShipped, Value: "false"}} + kept := componentOf("pkg:deb/debian/libc6@2.36") + kept.Properties = &[]cdx.Property{{Name: PropertyShipped, Value: "false"}} + + bom := bomOf(module, osPM, kept) + + filesets := filesetsOf(emptyIndex()) + filesets.OSPMFileListMissing = true + + Expect(MarkNotShipped(ctx, bom, filesets)).To(BeZero()) + for i := range *bom.Components { + _, found := shippedProperty(bom, i) + Expect(found).To(BeFalse(), "component %d keeps a verdict this image never made", i) + } + }) + + It("leaves the component's other properties alone", func(ctx SpecContext) { + comp := componentOf("pkg:pypi/present@1.0.0", fileOccurrence("/app/present.py", "here")) + comp.Properties = &[]cdx.Property{ + {Name: "werf:package:foundBy", Value: "pip"}, + {Name: PropertyShipped, Value: "false"}, + } + + bom := bomOf(comp) + + MarkNotShipped(ctx, bom, filesetsOf(indexOf(map[string]File{ + "/app/present.py": {Digest: digestOf("here"), Size: 4}, + }))) + + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(Equal([]cdx.Property{ + {Name: "werf:package:foundBy", Value: "pip"}, + })) + }) + + It("replaces a property the component already carries", func(ctx SpecContext) { + comp := componentOf("pkg:pypi/gone@2.0.0", fileOccurrence("/app/gone.py", "gone")) + comp.Properties = &[]cdx.Property{ + {Name: "werf:package:foundBy", Value: "pm"}, + {Name: PropertyShipped, Value: "true"}, + } + + bom := bomOf(comp) + + Expect(MarkNotShipped(ctx, bom, filesetsOf(emptyIndex()))).To(Equal(1)) + Expect(lo.FromPtr((*bom.Components)[0].Properties)).To(HaveLen(2)) + value, found := shippedProperty(bom, 0) + Expect(found).To(BeTrue()) + Expect(value).To(Equal("false")) + }) + + It("does nothing without an index of the image", func(ctx SpecContext) { + bom := bomOf(componentOf("pkg:pypi/gone@2.0.0", fileOccurrence("/app/gone.py", "gone"))) + + Expect(MarkNotShipped(ctx, bom, &Filesets{})).To(BeZero()) + Expect((*bom.Components)[0].Properties).To(BeNil()) + }) +}) diff --git a/pkg/sbom/shippedscope/occurrences.go b/pkg/sbom/shippedscope/occurrences.go index dd2ae8a17e..0b7f5f5041 100644 --- a/pkg/sbom/shippedscope/occurrences.go +++ b/pkg/sbom/shippedscope/occurrences.go @@ -26,7 +26,13 @@ import ( // // Occurrences the component already carries are replaced: a fileset is what the image // holds, and a scanner's guess at where it saw the package is not evidence of the same -// kind. +// kind. The one exception is a fileset naming files the image holds at none of them, over +// a component werf itself recorded occurrences for before: those digests are the only way +// left to find the bytes where this image put them, and an empty list over them would say +// the files are gone when they may only have moved. Over a component werf never recorded, +// the empty list is written, because there is nothing else that could show the files and +// "its own file list names files it does not hold" is what an image that installed a +// package and then deleted it looks like. // // Returns the number of components given occurrences. func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int { @@ -56,7 +62,9 @@ func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int { return } - setOccurrences(comp, sets) + if !setOccurrences(comp, sets) { + return + } recorded++ }) @@ -153,10 +161,40 @@ func canonicalName(ecosystem Ecosystem, name string) string { // content of its own to compare. const symlinkContext = "symlink" +const digestPrefix = "sha256:" + +// recordedOccurrences reads back what werf recorded of a component's files. An occurrence +// without a digest or the symlink marker came from somewhere else — a scanner noting where +// it saw a package, a document merged in — and says nothing about content; a component +// carrying none of werf's own is one nothing is known about, not one whose files were +// looked for and missed. +func recordedOccurrences(comp *cdx.Component) []Occurrence { + if comp.Evidence == nil || comp.Evidence.Occurrences == nil { + return nil + } + + recorded := *comp.Evidence.Occurrences + occurrences := make([]Occurrence, 0, len(recorded)) + for _, occurrence := range recorded { + switch { + case occurrence.AdditionalContext == symlinkContext: + occurrences = append(occurrences, Occurrence{Path: occurrence.Location, Symlink: true}) + case strings.HasPrefix(occurrence.AdditionalContext, digestPrefix): + occurrences = append(occurrences, Occurrence{Path: occurrence.Location, Digest: occurrence.AdditionalContext}) + } + } + + if len(recorded) > 0 && len(occurrences) == 0 { + return nil + } + + return occurrences +} + // setOccurrences records every file of every fileset of one component. npm installs a // package more than once under different paths, and each copy is a fileset of its own; -// what the component ships is all of them. -func setOccurrences(comp *cdx.Component, sets []Fileset) { +// what the component ships is all of them. Reports whether anything was recorded. +func setOccurrences(comp *cdx.Component, sets []Fileset) bool { contexts := make(map[string]string) for _, fileset := range sets { for file, entry := range fileset.Files { @@ -167,6 +205,10 @@ func setOccurrences(comp *cdx.Component, sets []Fileset) { } } + if len(contexts) == 0 && recordedOccurrences(comp) != nil { + return false + } + occurrences := make([]cdx.EvidenceOccurrence, 0, len(contexts)) for _, file := range slices.Sorted(maps.Keys(contexts)) { occurrences = append(occurrences, cdx.EvidenceOccurrence{ @@ -179,6 +221,8 @@ func setOccurrences(comp *cdx.Component, sets []Fileset) { comp.Evidence = &cdx.Evidence{} } comp.Evidence.Occurrences = &occurrences + + return true } func walkComponents(bom *cdx.BOM, visit func(comp *cdx.Component)) { diff --git a/pkg/sbom/shippedscope/occurrences_test.go b/pkg/sbom/shippedscope/occurrences_test.go index 8abf47d3d8..75121038be 100644 --- a/pkg/sbom/shippedscope/occurrences_test.go +++ b/pkg/sbom/shippedscope/occurrences_test.go @@ -103,6 +103,40 @@ var _ = Describe("RecordOccurrences", func() { Expect((*bom.Components)[0].Evidence).To(BeNil()) }) + It("keeps what a donor recorded when the image holds none of the files its manifest declares", func() { + bom := bomOf("pkg:pypi/requests@2.32.3") + (*bom.Components)[0].Evidence = &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + }} + + recorded := RecordOccurrences(bom, []Fileset{{ + Ecosystem: EcosystemPython, + Name: "requests", + Version: "2.32.3", + DeclaresFiles: true, + }}) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + }), "the digests a donor recorded are the only way left to find the bytes this image moved") + }) + + It("says the files were looked for when the image holds no declared file of a component no donor described", func() { + bom := bomOf("pkg:pypi/requests@2.32.3") + + recorded := RecordOccurrences(bom, []Fileset{{ + Ecosystem: EcosystemPython, + Name: "requests", + Version: "2.32.3", + DeclaresFiles: true, + }}) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{}), + "a package the image installed and then deleted looks exactly like this, and nothing else could show its files") + }) + It("records a component one of whose copies declares files", func() { bom := bomOf("pkg:npm/debug@4.3.4") @@ -192,7 +226,22 @@ var _ = Describe("RecordOccurrences", func() { }) Expect(recorded).To(Equal(1)) - Expect((*bom.Components)[0].Evidence.Occurrences).To(Equal(&[]cdx.EvidenceOccurrence{})) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{})) + }) + + It("keeps the occurrences werf recorded before over a fileset the image kept no file of, but replaces a scanner's", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + (*bom.Components)[0].Evidence = &cdx.Evidence{Occurrences: &[]cdx.EvidenceOccurrence{ + {Location: "/var/lib/dpkg/status"}, + }} + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", nil, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{}), + "where a scanner saw the package says nothing of its files, so there is nothing worth keeping over the image's own answer") }) It("leaves a component no fileset names unmarked, so nothing is known of its files", func() { diff --git a/test/e2e/sbom/_fixtures/inject/pip_partial_import/requirements.txt b/test/e2e/sbom/_fixtures/inject/pip_partial_import/requirements.txt new file mode 100644 index 0000000000..d80d9fc2a3 --- /dev/null +++ b/test/e2e/sbom/_fixtures/inject/pip_partial_import/requirements.txt @@ -0,0 +1 @@ +requests==2.32.3 diff --git a/test/e2e/sbom/_fixtures/inject/pip_partial_import/werf.yaml b/test/e2e/sbom/_fixtures/inject/pip_partial_import/werf.yaml new file mode 100644 index 0000000000..0ecfebda2d --- /dev/null +++ b/test/e2e/sbom/_fixtures/inject/pip_partial_import/werf.yaml @@ -0,0 +1,29 @@ +project: werf-test-e2e-sbom-inject-pip-partial-import +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: donor +from: registry.deckhouse.io/container-factory@sha256:7aac8d97a91ac233b19111aaad9d7bba12b975e620cfa9287b97e0003e4d0a71 +git: + - add: / + to: /app + stageDependencies: + packages: + - requirements.txt +packages: + - type: os-pm + spec: + - python==3.12.12 + - type: python-pip + workdir: /app +--- +image: app +from: registry.werf.io/werf/scratch:latest +import: + - image: donor + add: /app/requirements.txt + to: /imported/requirements.txt + before: setup diff --git a/test/e2e/sbom/pip_test.go b/test/e2e/sbom/pip_test.go index 3acf317e2a..ff0c9feb52 100644 --- a/test/e2e/sbom/pip_test.go +++ b/test/e2e/sbom/pip_test.go @@ -1,9 +1,12 @@ package e2e_build_test import ( + cdx "github.com/CycloneDX/cyclonedx-go" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" + "github.com/werf/werf/v3/pkg/sbom/shippedscope" sbomtest "github.com/werf/werf/v3/test/pkg/sbom" "github.com/werf/werf/v3/test/pkg/werf" ) @@ -85,4 +88,59 @@ var _ = Describe("SBOM python-pip packages", Label("e2e", "sbom", "pip", "simple } Expect(occurrences).To(ContainElement(HaveField("Location", HaveSuffix("/requests/__init__.py")))) }) + + It("marks a package of the donor whose files the importing image does not carry", func(ctx SpecContext) { + setupSbomBuildEnv() + + repoDirname := "repo_sbom_python_pip_partial_import" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/pip_partial_import") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{}}) + + bomOf := func(image string) *cdx.BOM { + return sbomtest.MustParseSBOMOutput(werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{image}}, + })) + } + + shippedProperty := func(comp *cdx.Component) (string, bool) { + for _, property := range lo.FromPtr(comp.Properties) { + if property.Name == shippedscope.PropertyShipped { + return property.Value, true + } + } + + return "", false + } + + donorBOM := bomOf("donor") + installed := sbomtest.FindComponent(donorBOM, "requests", "2.32.3") + Expect(installed).NotTo(BeNil(), "expected requests@2.32.3 in the BOM of the image that installs it") + _, marked := shippedProperty(installed) + Expect(marked).To(BeFalse(), "the image that installed requests holds its files, so it must not be marked") + + for _, comp := range lo.FromPtr(donorBOM.Components) { + _, marked := shippedProperty(&comp) + Expect(marked).To(BeFalse(), "the image that installed its packages ships them: %s@%s is marked", comp.Name, comp.Version) + } + + // The importing image takes one file of the donor and inherits the donor's whole BOM + // with it, requests included, while none of the files requests consists of came along. + appBOM := bomOf("app") + inherited := sbomtest.FindComponent(appBOM, "requests", "2.32.3") + Expect(inherited).NotTo(BeNil(), "expected requests@2.32.3 to be inherited into the BOM of the importing image") + value, marked := shippedProperty(inherited) + Expect(marked).To(BeTrue(), "requests is in the BOM of an image that carries none of its files, so it must be marked") + Expect(value).To(Equal("false")) + + // python is inherited the same way and is not in the importing image either, but the pm + // that installed it into the donor wrote no file list, so nothing can show where its + // files are and it is left unjudged rather than marked. + python := sbomtest.FindComponent(appBOM, "python", "3.12.12") + Expect(python).NotTo(BeNil(), "expected python@3.12.12 to be inherited into the BOM of the importing image") + _, marked = shippedProperty(python) + Expect(marked).To(BeFalse(), "an os-pm package whose files no file list names is one nothing is known about") + }) })