diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index e69ebf939d..c73a706f0a 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "maps" "os" "path/filepath" @@ -13,6 +14,8 @@ import ( "time" cdx "github.com/CycloneDX/cyclonedx-go" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" packageurl "github.com/package-url/packageurl-go" "github.com/samber/lo" "github.com/sigstore/sigstore/pkg/signature" @@ -32,7 +35,9 @@ import ( "github.com/werf/werf/v3/pkg/sbom/managedinput" osPm "github.com/werf/werf/v3/pkg/sbom/packages/os_pm" "github.com/werf/werf/v3/pkg/sbom/scanner" + "github.com/werf/werf/v3/pkg/sbom/shippedscope" "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/tmp_manager" "github.com/werf/werf/v3/pkg/werf" "github.com/werf/werf/v3/pkg/werf/global_warnings" ) @@ -202,6 +207,8 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) + recordShippedFiles(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name) + resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { return fmt.Errorf("serialize BOM: %w", err) @@ -221,6 +228,87 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, }) } +// recordShippedFiles records, on every component an ecosystem installed into the image, +// which files of the image are that component's. An image built from this one can then +// ask of an inherited component whether its files are still there, instead of trusting +// that a component its base declared came along. +// +// Runs last, once the BOM holds the components it will ship with: Canonicalize folds +// duplicate components together and drops an evidence list that comes out empty, which is +// exactly the "its files were looked for and none were found" this writes. +// +// The image is read out of the backend the step pulled it into, not from the registry a +// second time: the read is then as long as a save to local disk, and what can fail it is +// the host — an image that is not there, a disk that is full — rather than the network. +// +// Nothing here fails the build. A BOM without these records is the BOM werf wrote before +// it kept them, and a reader that finds no record of a component's files keeps the +// component, so the worst an unreadable image costs is the SBOM of the image built from +// this one being as complete as it used to be. +func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) { + if !shippedscope.HasFileBackedComponents(bom) { + return + } + + img, cleanup, err := savedImage(ctx, backend, imageName) + if err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read the built image %s; the SBOM records none of the files its components consist of: %s\n", werfImgName, imageName, err) + + return + } + defer cleanup() + + filesets, err := shippedscope.CollectFilesets(ctx, img) + if err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read which files the components of the built image %s consist of; the SBOM records none of them: %s\n", werfImgName, imageName, err) + + return + } + + recorded := shippedscope.RecordOccurrences(bom, filesets.Filesets) + logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) +} + +// savedImage reads the image out of the backend as a docker-save tarball. The tarball is +// read more than once, once for its manifest and once per layer, which a stream cannot +// give, so it goes to a file first, the same way MutateAndPushImage does. The returned +// cleanup removes that file. +func savedImage(ctx context.Context, backend imageSaver, imageName string) (v1.Image, func(), error) { + stream, err := backend.SaveImageToStream(ctx, imageName) + if err != nil { + return nil, nil, fmt.Errorf("save image: %w", err) + } + defer stream.Close() + + file, err := tmp_manager.TempFile("werf-sbom-shipped-files-*.tar") + if err != nil { + return nil, nil, fmt.Errorf("create temp file: %w", err) + } + cleanup := func() { _ = os.Remove(file.Name()) } + + if _, err := io.Copy(file, stream); err != nil { + _ = file.Close() + cleanup() + + return nil, nil, fmt.Errorf("write saved image to %s: %w", file.Name(), err) + } + + if err := file.Close(); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("close %s: %w", file.Name(), err) + } + + img, err := tarball.ImageFromPath(file.Name(), nil) + if err != nil { + cleanup() + + return nil, nil, fmt.Errorf("read saved image %s: %w", file.Name(), err) + } + + return img, cleanup, nil +} + // restoreImageMetadata sets the BOM's top-level component to the scanned image while // keeping any syft-provided metadata (tools, timestamp). A directory source reports the // temporary scan directory as its component, so it must be replaced; the dependency edges @@ -460,7 +548,7 @@ func (step *sbomStep) recordDeclaredPackages(ctx context.Context, bom *cdx.BOM, return nil } -const sbomArtifactFormatVersion = "10" +const sbomArtifactFormatVersion = "11" // calculateStableChecksum computes the SBOM artifact cache checksum. Together with the // parent stage digest it forms the cache key: a previously attached SBOM is reused only diff --git a/pkg/build/sbom_step_saved_image.go b/pkg/build/sbom_step_saved_image.go new file mode 100644 index 0000000000..0abd30df29 --- /dev/null +++ b/pkg/build/sbom_step_saved_image.go @@ -0,0 +1,12 @@ +package build + +import ( + "context" + "io" +) + +// Kept out of sbom_step.go: the mockgen directive there generates a mock for every +// interface in its source file, and a test double for this one is a stub, not a mock. +type imageSaver interface { + SaveImageToStream(ctx context.Context, imageName string) (io.ReadCloser, error) +} diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go new file mode 100644 index 0000000000..c92b9d3b82 --- /dev/null +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -0,0 +1,245 @@ +package build + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "path/filepath" + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/static" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + "github.com/werf/logboek" + "github.com/werf/werf/v3/pkg/werf" +) + +type savedImageBackend struct { + tarball []byte + err error + saved []string +} + +var _ imageSaver = (*savedImageBackend)(nil) + +func (b *savedImageBackend) SaveImageToStream(_ context.Context, imageName string) (io.ReadCloser, error) { + b.saved = append(b.saved, imageName) + if b.err != nil { + return nil, b.err + } + + return io.NopCloser(bytes.NewReader(b.tarball)), nil +} + +var _ = Describe("recordShippedFiles", func() { + tarOf := func(files, symlinks map[string]string) []byte { + var buf bytes.Buffer + writer := tar.NewWriter(&buf) + for _, path := range lo.Keys(files) { + body := files[path] + Expect(writer.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(path, "/"), + Typeflag: tar.TypeReg, + Size: int64(len(body)), + Mode: 0o644, + })).To(Succeed()) + _, err := io.WriteString(writer, body) + Expect(err).To(Succeed()) + } + for _, path := range lo.Keys(symlinks) { + Expect(writer.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(path, "/"), + Typeflag: tar.TypeSymlink, + Linkname: symlinks[path], + Mode: 0o777, + })).To(Succeed()) + } + Expect(writer.Close()).To(Succeed()) + + return buf.Bytes() + } + + // Buildah saves layers uncompressed and Docker may save them either way; tarball.Write + // copies what Compressed() gives, so the layer type decides which the saved tarball holds. + gzippedLayer := func(contents []byte) v1.Layer { + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(contents)), nil + }) + Expect(err).To(Succeed()) + + return layer + } + uncompressedLayer := func(contents []byte) v1.Layer { + return static.NewLayer(contents, types.DockerUncompressedLayer) + } + + backendSaving := func(layerOf func([]byte) v1.Layer, files, symlinks map[string]string) *savedImageBackend { + img, err := mutate.AppendLayers(empty.Image, layerOf(tarOf(files, symlinks))) + Expect(err).To(Succeed()) + + tag, err := name.NewTag("werf-stages:app") + Expect(err).To(Succeed()) + + var saved bytes.Buffer + Expect(tarball.Write(tag, img, &saved)).To(Succeed()) + + return &savedImageBackend{tarball: saved.Bytes()} + } + + backendWith := func(files, symlinks map[string]string) *savedImageBackend { + return backendSaving(gzippedLayer, files, symlinks) + } + + digestOf := func(body string) string { + sum := sha256.Sum256([]byte(body)) + + return "sha256:" + hex.EncodeToString(sum[:]) + } + + logged := func(ctx SpecContext, record func(ctx context.Context)) string { + var out bytes.Buffer + logger := logboek.NewLogger(&out, &out) + logger.Streams().DisableLineWrapping() + record(logboek.NewContext(ctx, logger)) + + return out.String() + } + + savedTarballs := func() []string { + matches, err := filepath.Glob(filepath.Join(werf.GetTmpDir(), "werf-sbom-shipped-files-*.tar")) + Expect(err).To(Succeed()) + + return matches + } + + BeforeEach(func() { + Expect(werf.Init(GinkgoT().TempDir(), "")).To(Succeed()) + }) + + DescribeTable("records the files of the image's own packages onto its BOM", func(ctx SpecContext, layerOf func([]byte) v1.Layer) { + backend := backendSaving(layerOf, map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/var/lib/pm/files.json": `{"zlib":{"version":"1.3.1","files":{"/usr/lib/libz.so.1.3.1":"sha256:recorded-at-unpack"},"symlinks":{"/usr/lib/libz.so.1":"libz.so.1.3.1"}}}`, + "/usr/lib/libz.so.1.3.1": "zlib body", + "/usr/lib/python3/requests-2.32.3.dist-info/RECORD": "requests/__init__.py,sha256=x,12\n", + "/usr/lib/python3/requests/__init__.py": "requests body", + }, map[string]string{ + "/usr/lib/libz.so.1": "libz.so.1.3.1", + }) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1?containerfactoryversion=1"}, + {BOMRef: "requests", Name: "requests", PackageURL: "pkg:pypi/requests@2.32.3"}, + {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, + }} + + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + + Expect(backend.saved).To(Equal([]string{"werf-stages:app"})) + components := *bom.Components + Expect(lo.FromPtr(components[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1", AdditionalContext: "symlink"}, + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("zlib body")}, + })) + Expect(lo.FromPtr(components[1].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("requests body")}, + })) + Expect(components[2].Evidence).To(BeNil()) + Expect(savedTarballs()).To(BeEmpty()) + }, + Entry("saved with gzipped layers", gzippedLayer), + Entry("saved with uncompressed layers, as Buildah saves them", uncompressedLayer), + ) + + It("leaves os-pm packages unmarked when the image was installed by a pm that wrote no file list", func(ctx SpecContext) { + backend := backendWith(map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/usr/lib/libz.so.1.3.1": "zlib body", + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("warns and leaves the BOM as it was when the backend cannot save the image", func(ctx SpecContext) { + backend := &savedImageBackend{err: errors.New("no such image")} + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + }) + + Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) + Expect(out).To(ContainSubstring("no such image")) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("warns and leaves the BOM as it was when the saved image is not an image", func(ctx SpecContext) { + backend := &savedImageBackend{tarball: []byte("not a tarball")} + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + }) + + Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + Expect(savedTarballs()).To(BeEmpty()) + }) + + It("warns and leaves the BOM as it was when the image's file lists cannot be read", func(ctx SpecContext) { + backend := backendWith(map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/var/lib/pm/files.json": `not json at all`, + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + }) + + Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image werf-stages:app consist of")) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + Expect(savedTarballs()).To(BeEmpty()) + }) + + It("warns nothing and does not save the image when no component could be given files", func(ctx SpecContext) { + backend := &savedImageBackend{err: errors.New("must not be called")} + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "yaml", Name: "gopkg.in/yaml.v3", PackageURL: "pkg:golang/gopkg.in/yaml.v3@3.0.1"}, + {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + }) + + Expect(out).To(BeEmpty()) + Expect(backend.saved).To(BeEmpty()) + }) +}) diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go index b49fe0576c..1ba1efaab0 100644 --- a/pkg/sbom/shippedscope/fileset.go +++ b/pkg/sbom/shippedscope/fileset.go @@ -28,6 +28,11 @@ type Fileset struct { Version string Files map[string]File Symlinks map[string]string + // DeclaresFiles tells whether the component named any path at all. One that names none + // — a metapackage, a distribution whose record is empty — can be shown neither present + // nor absent by its files, which is a different thing from one whose declared files are + // all gone. + DeclaresFiles bool } type Filesets struct { diff --git a/pkg/sbom/shippedscope/javascript_fileset.go b/pkg/sbom/shippedscope/javascript_fileset.go index 705c6e2552..1be073c01d 100644 --- a/pkg/sbom/shippedscope/javascript_fileset.go +++ b/pkg/sbom/shippedscope/javascript_fileset.go @@ -44,14 +44,16 @@ func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset root := path.Dir(manifest) - files, symlinks := collect(index, packageTreePaths(index, root)) + declared := packageTreePaths(index, root) + files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemJavaScript, - Name: declaration.Name, - Version: declaration.Version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemJavaScript, + Name: declaration.Name, + Version: declaration.Version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/lua_fileset.go b/pkg/sbom/shippedscope/lua_fileset.go index 4c8c6e053f..5ec126beea 100644 --- a/pkg/sbom/shippedscope/lua_fileset.go +++ b/pkg/sbom/shippedscope/lua_fileset.go @@ -62,11 +62,12 @@ func luaFilesets(ctx context.Context, index *FileIndex, manifests map[string][]b files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemLua, - Name: name, - Version: version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemLua, + Name: name, + Version: version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/lua_fileset_test.go b/pkg/sbom/shippedscope/lua_fileset_test.go index 7a499cebef..e38bc43e92 100644 --- a/pkg/sbom/shippedscope/lua_fileset_test.go +++ b/pkg/sbom/shippedscope/lua_fileset_test.go @@ -111,4 +111,18 @@ var _ = Describe("CollectFilesets, Lua", func() { Expect(filesets.Filesets).To(BeEmpty()) }) + + It("says a rock whose manifest names no file declares none", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/meta/1.0-1/rock_manifest", "rock_manifest = {\n lua = {\n }\n}\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + meta := filesetOf(filesets, EcosystemLua, "meta", "1.0-1") + Expect(meta).NotTo(BeNil()) + Expect(meta.DeclaresFiles).To(BeFalse()) + }) }) diff --git a/pkg/sbom/shippedscope/occurrences.go b/pkg/sbom/shippedscope/occurrences.go new file mode 100644 index 0000000000..dd2ae8a17e --- /dev/null +++ b/pkg/sbom/shippedscope/occurrences.go @@ -0,0 +1,197 @@ +package shippedscope + +import ( + "maps" + "slices" + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + packageurl "github.com/package-url/packageurl-go" + "github.com/samber/lo" +) + +// RecordOccurrences writes the files of each fileset onto the component that owns them, as +// evidence.occurrences: the path the file has in this image as location, and what is known +// of its content as additionalContext — the digest of a regular file, or symlinkContext +// for a symlink, which holds no bytes of its own. Every occurrence written here carries +// one or the other, so a reader can tell werf's own evidence from an occurrence that +// arrived in a merged document and means something else entirely. +// +// A component no fileset names is left as it was, and so is one whose filesets all declare +// no files at all. Nothing is then known about its files — its ecosystem records no file +// lists, it was installed in a way none of them reads, or it is a metapackage that ships +// nothing — and an empty occurrence list would instead claim its files were looked for and +// none were found. That is what keeps os-pm components unmarked when the image was +// installed by a pm too old to write a file list. +// +// Occurrences the component already carries are replaced: a fileset is what the image +// holds, and a scanner's guess at where it saw the package is not evidence of the same +// kind. +// +// Returns the number of components given occurrences. +func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int { + if bom == nil || len(filesets) == 0 { + return 0 + } + + owned := make(map[componentKey][]Fileset, len(filesets)) + for _, fileset := range filesets { + key := componentKey{ + ecosystem: fileset.Ecosystem, + name: canonicalName(fileset.Ecosystem, fileset.Name), + version: fileset.Version, + } + owned[key] = append(owned[key], fileset) + } + + recorded := 0 + walkComponents(bom, func(comp *cdx.Component) { + key, ok := keyOfComponent(comp) + if !ok { + return + } + + sets, ok := owned[key] + if !ok || !lo.SomeBy(sets, func(fileset Fileset) bool { return fileset.DeclaresFiles }) { + return + } + + setOccurrences(comp, sets) + recorded++ + }) + + return recorded +} + +// HasFileBackedComponents tells whether the BOM holds a component any ecosystem could +// record files for. Reading an image to find out costs a full download of its layers, and +// an image whose components are all Go modules, deb packages or anything else no collector +// reads has nothing to gain from it. +func HasFileBackedComponents(bom *cdx.BOM) bool { + if bom == nil { + return false + } + + found := false + walkComponents(bom, func(comp *cdx.Component) { + if _, ok := keyOfComponent(comp); ok { + found = true + } + }) + + return found +} + +// componentKey is what a fileset and a component have to agree on to be the same install: +// one ecosystem's idea of a package, not another's. Two ecosystems naming a package alike +// are two packages. +type componentKey struct { + ecosystem Ecosystem + name string + version string +} + +func keyOfComponent(comp *cdx.Component) (componentKey, bool) { + if comp.PackageURL == "" { + return componentKey{}, false + } + + purl, err := packageurl.FromString(comp.PackageURL) + if err != nil { + return componentKey{}, false + } + + ecosystem, ok := purlEcosystems[purl.Type] + if !ok { + return componentKey{}, false + } + + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + purl.Name + } + + return componentKey{ + ecosystem: ecosystem, + name: canonicalName(ecosystem, name), + version: purl.Version, + }, true +} + +// purlEcosystems maps the purl type of a component to the ecosystem whose filesets can +// describe it. os-pm packages are pkg:generic, the type werf gives what its own pm +// installs; a deb or rpm component comes from a package manager whose file list nothing +// here reads, so it has no fileset and stays unmarked. +// +// pkg:generic is not werf's alone — syft gives it to a runtime it recognizes by its binary +// — but taking a fileset needs the same name and version as a package the pm recorded, and +// a component agreeing with a pm entry on both is that package. +var purlEcosystems = map[string]Ecosystem{ + packageurl.TypeGeneric: EcosystemOSPM, + packageurl.TypePyPi: EcosystemPython, + packageurl.TypeNPM: EcosystemJavaScript, + luarocksPurlType: EcosystemLua, +} + +const luarocksPurlType = "luarocks" + +// canonicalName spells a name the way its ecosystem considers canonical, so that a +// fileset and a component that mean the same package are keyed alike: PEP 503 folds +// separator runs and case, npm names are case-insensitive. +func canonicalName(ecosystem Ecosystem, name string) string { + switch ecosystem { + case EcosystemPython: + return normalizePythonName(name) + case EcosystemJavaScript: + return strings.ToLower(name) + default: + return name + } +} + +// symlinkContext marks an occurrence as a symlink, which has a path in the image but no +// content of its own to compare. +const symlinkContext = "symlink" + +// setOccurrences records every file of every fileset of one component. npm installs a +// package more than once under different paths, and each copy is a fileset of its own; +// what the component ships is all of them. +func setOccurrences(comp *cdx.Component, sets []Fileset) { + contexts := make(map[string]string) + for _, fileset := range sets { + for file, entry := range fileset.Files { + contexts[file] = entry.Digest + } + for link := range fileset.Symlinks { + contexts[link] = symlinkContext + } + } + + occurrences := make([]cdx.EvidenceOccurrence, 0, len(contexts)) + for _, file := range slices.Sorted(maps.Keys(contexts)) { + occurrences = append(occurrences, cdx.EvidenceOccurrence{ + Location: file, + AdditionalContext: contexts[file], + }) + } + + if comp.Evidence == nil { + comp.Evidence = &cdx.Evidence{} + } + comp.Evidence.Occurrences = &occurrences +} + +func walkComponents(bom *cdx.BOM, visit func(comp *cdx.Component)) { + var walk func(components []cdx.Component) + walk = func(components []cdx.Component) { + for i := range components { + visit(&components[i]) + walk(lo.FromPtr(components[i].Components)) + } + } + + walk(lo.FromPtr(bom.Components)) + if bom.Metadata != nil && bom.Metadata.Component != nil { + walk(lo.FromPtr(bom.Metadata.Component.Components)) + } +} diff --git a/pkg/sbom/shippedscope/occurrences_test.go b/pkg/sbom/shippedscope/occurrences_test.go new file mode 100644 index 0000000000..8abf47d3d8 --- /dev/null +++ b/pkg/sbom/shippedscope/occurrences_test.go @@ -0,0 +1,313 @@ +package shippedscope + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" +) + +var _ = Describe("RecordOccurrences", func() { + componentOf := func(purl string) cdx.Component { + return cdx.Component{BOMRef: purl, Name: purl, PackageURL: purl} + } + + bomOf := func(purls ...string) *cdx.BOM { + components := make([]cdx.Component, 0, len(purls)) + for _, purl := range purls { + components = append(components, componentOf(purl)) + } + + return &cdx.BOM{Components: &components} + } + + occurrencesOf := func(bom *cdx.BOM, index int) []cdx.EvidenceOccurrence { + comp := (*bom.Components)[index] + if comp.Evidence == nil { + return nil + } + + return lo.FromPtr(comp.Evidence.Occurrences) + } + + fileset := func(ecosystem Ecosystem, name, version string, files, symlinks map[string]string) Fileset { + entries := make(map[string]File, len(files)) + for file, body := range files { + entries[file] = File{Digest: digestOf(body), Size: int64(len(body))} + } + + return Fileset{ + Ecosystem: ecosystem, + Name: name, + Version: version, + Files: entries, + Symlinks: symlinks, + DeclaresFiles: true, + } + } + + It("records the files of a component of each ecosystem", func() { + bom := bomOf( + "pkg:generic/zlib@1.3.1?containerfactoryversion=1", + "pkg:pypi/requests@2.32.3", + "pkg:npm/debug@4.3.4", + "pkg:luarocks/luasocket@3.1.0-1", + ) + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + fileset(EcosystemPython, "requests", "2.32.3", map[string]string{"/usr/lib/python3/requests/__init__.py": "r"}, nil), + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + fileset(EcosystemLua, "luasocket", "3.1.0-1", map[string]string{"/usr/lib/lua/socket/core.so": "l"}, nil), + }) + + Expect(recorded).To(Equal(4)) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + Expect(occurrencesOf(bom, 1)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + })) + Expect(occurrencesOf(bom, 2)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/app/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + })) + Expect(occurrencesOf(bom, 3)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/lua/socket/core.so", AdditionalContext: digestOf("l")}, + })) + }) + + It("marks a symlink as one instead of leaving it to be told apart by a missing digest", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", + map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, + map[string]string{"/usr/lib/libz.so.1": "libz.so.1.3.1"}, + ), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1", AdditionalContext: "symlink"}, + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + }) + + It("leaves a component that declares no files alone, instead of saying none were found", func() { + bom := bomOf("pkg:generic/build-essential@12.9") + + recorded := RecordOccurrences(bom, []Fileset{ + {Ecosystem: EcosystemOSPM, Name: "build-essential", Version: "12.9"}, + }) + + Expect(recorded).To(BeZero()) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("records a component one of whose copies declares files", func() { + bom := bomOf("pkg:npm/debug@4.3.4") + + recorded := RecordOccurrences(bom, []Fileset{ + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "4.3.4"}, + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("leaves a component of another version alone", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.2.13", map[string]string{"/usr/lib/libz.so.1.2.13": "old"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) + + It("leaves a component of another ecosystem alone, however alike it is named", func() { + bom := bomOf("pkg:pypi/debug@4.3.4") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) + + It("matches a PyPI component whose purl keeps a separator the install folds", func() { + bom := bomOf("pkg:pypi/ruamel.yaml@0.18.6") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemPython, "ruamel-yaml", "0.18.6", map[string]string{"/usr/lib/python3/ruamel/yaml/__init__.py": "y"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("matches an npm component whose package.json spells the name in mixed case", func() { + bom := bomOf("pkg:npm/JSONStream@1.3.5") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "JSONStream", "1.3.5", map[string]string{"/app/node_modules/JSONStream/index.js": "j"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("matches a scoped npm component to the package tree it was installed into", func() { + bom := bomOf("pkg:npm/%40babel/core@7.24.0") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "@babel/core", "7.24.0", map[string]string{"/app/node_modules/@babel/core/index.js": "b"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("gives a component installed more than once the files of every copy", func() { + bom := bomOf("pkg:npm/debug@4.3.4") + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/ms/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/app/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + {Location: "/app/node_modules/ms/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + })) + }) + + It("says the files were looked for when a fileset names a component but the image kept none", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", nil, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect((*bom.Components)[0].Evidence.Occurrences).To(Equal(&[]cdx.EvidenceOccurrence{})) + }) + + It("leaves a component no fileset names unmarked, so nothing is known of its files", func() { + bom := bomOf("pkg:deb/debian/libc6@2.36", "pkg:golang/gopkg.in/yaml.v3@3.0.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "libc6", "2.36", map[string]string{"/lib/libc.so.6": "c"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + Expect(occurrencesOf(bom, 1)).To(BeNil()) + }) + + It("records a component nested under another", func() { + nested := []cdx.Component{componentOf("pkg:generic/zlib@1.3.1")} + outer := componentOf("pkg:generic/git@2.50.1") + outer.Components = &nested + bom := &cdx.BOM{Components: &[]cdx.Component{outer}} + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(lo.FromPtr((*(*bom.Components)[0].Components)[0].Evidence.Occurrences)).To(HaveLen(1)) + }) + + It("records a component listed under the image's own component", func() { + nested := []cdx.Component{componentOf("pkg:generic/zlib@1.3.1")} + bom := &cdx.BOM{Metadata: &cdx.Metadata{Component: &cdx.Component{ + Type: cdx.ComponentTypeContainer, + Components: &nested, + }}} + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(lo.FromPtr(nested[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + }) + + It("replaces the occurrences a scanner guessed with the files the image holds", func() { + bom := bomOf("pkg:pypi/requests@2.32.3") + (*bom.Components)[0].Evidence = &cdx.Evidence{ + Occurrences: &[]cdx.EvidenceOccurrence{{Location: "/usr/lib/python3/requests-2.32.3.dist-info/RECORD"}}, + } + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemPython, "requests", "2.32.3", map[string]string{"/usr/lib/python3/requests/__init__.py": "r"}, nil), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + })) + }) + + It("keeps the evidence a component already carries of another kind", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + (*bom.Components)[0].Evidence = &cdx.Evidence{ + Identity: &[]cdx.EvidenceIdentity{{Field: cdx.EvidenceIdentityFieldTypeCPE}}, + } + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(lo.FromPtr((*bom.Components)[0].Evidence.Identity)).To(HaveLen(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("does nothing to a BOM when the image yielded no filesets", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + Expect(RecordOccurrences(bom, nil)).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) +}) + +var _ = Describe("HasFileBackedComponents", func() { + bomOf := func(purls ...string) *cdx.BOM { + components := make([]cdx.Component, 0, len(purls)) + for _, purl := range purls { + components = append(components, cdx.Component{BOMRef: purl, Name: purl, PackageURL: purl}) + } + + return &cdx.BOM{Components: &components} + } + + It("finds a component of each ecosystem that records file lists", func() { + Expect(HasFileBackedComponents(bomOf("pkg:generic/zlib@1.3.1"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:pypi/requests@2.32.3"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:npm/debug@4.3.4"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:luarocks/luasocket@3.1.0-1"))).To(BeTrue()) + }) + + It("finds nothing in a BOM whose components no ecosystem here records", func() { + Expect(HasFileBackedComponents(bomOf( + "pkg:golang/gopkg.in/yaml.v3@3.0.1", + "pkg:deb/debian/libc6@2.36", + "pkg:cargo/serde@1.0.0", + ))).To(BeFalse()) + }) + + It("finds a component nested under another", func() { + nested := []cdx.Component{{BOMRef: "z", PackageURL: "pkg:generic/zlib@1.3.1"}} + outer := cdx.Component{BOMRef: "g", PackageURL: "pkg:golang/x@1", Components: &nested} + + Expect(HasFileBackedComponents(&cdx.BOM{Components: &[]cdx.Component{outer}})).To(BeTrue()) + }) + + It("finds nothing in an empty BOM", func() { + Expect(HasFileBackedComponents(&cdx.BOM{})).To(BeFalse()) + }) +}) diff --git a/pkg/sbom/shippedscope/os_pm_fileset.go b/pkg/sbom/shippedscope/os_pm_fileset.go index f97545ac43..f3a18b5121 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset.go +++ b/pkg/sbom/shippedscope/os_pm_fileset.go @@ -62,11 +62,12 @@ func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][] files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemOSPM, - Name: name, - Version: entry.Version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemOSPM, + Name: name, + Version: entry.Version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/os_pm_fileset_test.go b/pkg/sbom/shippedscope/os_pm_fileset_test.go index 3068cc3bd8..55d5e075a5 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset_test.go +++ b/pkg/sbom/shippedscope/os_pm_fileset_test.go @@ -65,6 +65,21 @@ var _ = Describe("CollectFilesets, os-pm", func() { git := filesetOf(filesets, EcosystemOSPM, "git", "2.50.1") Expect(git).NotTo(BeNil()) Expect(git.Files).To(HaveLen(1)) + Expect(git.DeclaresFiles).To(BeTrue()) + }) + + It("tells a package that recorded no files from one whose recorded files are gone", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("var/lib/pm/files.json", + `{"build-essential": {"version": "12.9"}, "git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead"}}}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesetOf(filesets, EcosystemOSPM, "build-essential", "12.9").DeclaresFiles).To(BeFalse()) + Expect(filesetOf(filesets, EcosystemOSPM, "git", "2.50.1").DeclaresFiles).To(BeTrue()) }) It("reads a file list the image reaches through a hardlink", func(ctx SpecContext) { diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go index a93f11c3b2..f01dda37b8 100644 --- a/pkg/sbom/shippedscope/python_fileset.go +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -50,11 +50,12 @@ func pythonFilesets(ctx context.Context, index *FileIndex, manifests map[string] files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemPython, - Name: normalizePythonName(name), - Version: version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemPython, + Name: normalizePythonName(name), + Version: version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go index 30053a9ab0..f0f5ae7ace 100644 --- a/pkg/sbom/shippedscope/python_fileset_test.go +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -123,6 +123,22 @@ var _ = Describe("CollectFilesets, Python", func() { stub := filesetOf(filesets, EcosystemPython, "stub", "1.0") Expect(stub).NotTo(BeNil()) Expect(stub.Files).To(BeEmpty()) + Expect(stub.DeclaresFiles).To(BeFalse()) + }) + + It("says a distribution declares files when its record names some, however few survived", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("site/gone-1.0.dist-info/RECORD", "gone/__init__.py,sha256=x,12\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + gone := filesetOf(filesets, EcosystemPython, "gone", "1.0") + Expect(gone).NotTo(BeNil()) + Expect(gone.Files).To(BeEmpty()) + Expect(gone.DeclaresFiles).To(BeTrue()) }) It("reads a record whose path is quoted because it holds a comma", func(ctx SpecContext) { diff --git a/test/e2e/sbom/pip_test.go b/test/e2e/sbom/pip_test.go index 85513f7163..3acf317e2a 100644 --- a/test/e2e/sbom/pip_test.go +++ b/test/e2e/sbom/pip_test.go @@ -52,4 +52,37 @@ var _ = Describe("SBOM python-pip packages", Label("e2e", "sbom", "pip", "simple Expect(requests).NotTo(BeNil(), "expected requests@2.32.3 in BOM: the project pip.py shadowed the installer and the packages stage reported success without installing anything") }) + + It("records which files of the image each pip package consists of", func(ctx SpecContext) { + setupSbomBuildEnv() + + repoDirname := "repo_sbom_python_pip_shipped_files" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/pip_simple") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{}}) + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"app"}, + }, + }) + + bom := sbomtest.MustParseSBOMOutput(sbomOut) + requests := sbomtest.FindComponent(bom, "requests", "2.32.3") + Expect(requests).NotTo(BeNil(), "expected requests@2.32.3 in BOM") + Expect(requests.Evidence).NotTo(BeNil(), "requests carries no evidence: the files it consists of were not recorded") + Expect(requests.Evidence.Occurrences).NotTo(BeNil(), "requests carries no occurrences: the files it consists of were not recorded") + + occurrences := *requests.Evidence.Occurrences + Expect(occurrences).NotTo(BeEmpty(), "requests is installed, so its files are in the image") + for _, occurrence := range occurrences { + Expect(occurrence.Location).To(HavePrefix("/"), "an occurrence names an absolute path in the image") + Expect(occurrence.Location).To(ContainSubstring("requests"), "an occurrence of requests names one of its files") + Expect(occurrence.AdditionalContext).To(Or(MatchRegexp(`^sha256:[0-9a-f]{64}$`), Equal("symlink")), + "every occurrence werf writes carries the file digest or marks a symlink") + } + Expect(occurrences).To(ContainElement(HaveField("Location", HaveSuffix("/requests/__init__.py")))) + }) })