From fac588fe4a36141a41b55f94732ad6ac4cb23370 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 11:15:51 +0300 Subject: [PATCH 1/6] feat(sbom): record which files each component of an image consists of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An image's SBOM says a component is there; it does not say which files are that component's. A child image built FROM this one, or importing files out of it, inherits the component list wholesale and has nothing to check it against, so a component whose files never arrived stays in the child's SBOM and is reported as its attack surface. Every image whose SBOM werf generates now carries, for each component its ecosystems installed, the files that component put into it: evidence.occurrences with the path as location and what is known of the content as additionalContext — the digest of a regular file, "symlink" for a symlink. Four ecosystems record file lists: os-pm, python, javascript, lua. A component none of them names keeps no occurrences at all, which is what a reader has to read as "nothing known", and the same holds for a component that names no file of its own — a metapackage, a distribution whose record is empty. An empty occurrence list means the opposite: the files were looked for and none were found. The writing happens after the BOM is canonicalized, because canonicalization folds duplicate components together and drops an evidence list that comes out empty, which is exactly the "none were found" this records. The SBOM artifact format version goes to 11, without which every image with an SBOM already attached would keep serving an occurrence-free one from the cache. Signed-off-by: Radmir Khurum --- pkg/build/sbom_step.go | 32 ++- pkg/build/sbom_step_shipped_files_test.go | 147 ++++++++++ pkg/oci/artifact/store.go | 17 ++ pkg/sbom/shippedscope/fileset.go | 5 + pkg/sbom/shippedscope/javascript_fileset.go | 14 +- pkg/sbom/shippedscope/lua_fileset.go | 11 +- pkg/sbom/shippedscope/lua_fileset_test.go | 14 + pkg/sbom/shippedscope/occurrences.go | 176 ++++++++++++ pkg/sbom/shippedscope/occurrences_test.go | 276 +++++++++++++++++++ pkg/sbom/shippedscope/os_pm_fileset.go | 11 +- pkg/sbom/shippedscope/os_pm_fileset_test.go | 15 + pkg/sbom/shippedscope/python_fileset.go | 11 +- pkg/sbom/shippedscope/python_fileset_test.go | 16 ++ 13 files changed, 723 insertions(+), 22 deletions(-) create mode 100644 pkg/build/sbom_step_shipped_files_test.go create mode 100644 pkg/sbom/shippedscope/occurrences.go create mode 100644 pkg/sbom/shippedscope/occurrences_test.go diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index e69ebf939d..efb081830a 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -32,6 +32,7 @@ import ( "github.com/werf/werf/v3/pkg/sbom/managedinput" osPm "github.com/werf/werf/v3/pkg/sbom/packages/os_pm" "github.com/werf/werf/v3/pkg/sbom/scanner" + "github.com/werf/werf/v3/pkg/sbom/shippedscope" "github.com/werf/werf/v3/pkg/storage" "github.com/werf/werf/v3/pkg/werf" "github.com/werf/werf/v3/pkg/werf/global_warnings" @@ -202,6 +203,10 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) + if err := recordShippedFiles(ctx, resultBOM, store, parentDigest); err != nil { + return err + } + resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { return fmt.Errorf("serialize BOM: %w", err) @@ -221,6 +226,31 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, }) } +// recordShippedFiles records, on every component an ecosystem installed into the image, +// which files of the image are that component's. An image built from this one can then +// ask of an inherited component whether its files are still there, instead of trusting +// that a component its base declared came along. +// +// Runs last, once the BOM holds the components it will ship with: Canonicalize folds +// duplicate components together and drops an evidence list that comes out empty, which is +// exactly the "its files were looked for and none were found" this writes. +func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, parentDigest string) error { + img, err := store.ParentImage(ctx, parentDigest) + if err != nil { + return fmt.Errorf("read built image: %w", err) + } + + filesets, err := shippedscope.CollectFilesets(ctx, img) + if err != nil { + return fmt.Errorf("collect component filesets: %w", err) + } + + recorded := shippedscope.RecordOccurrences(bom, filesets.Filesets) + logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) + + return nil +} + // restoreImageMetadata sets the BOM's top-level component to the scanned image while // keeping any syft-provided metadata (tools, timestamp). A directory source reports the // temporary scan directory as its component, so it must be replaced; the dependency edges @@ -460,7 +490,7 @@ func (step *sbomStep) recordDeclaredPackages(ctx context.Context, bom *cdx.BOM, return nil } -const sbomArtifactFormatVersion = "10" +const sbomArtifactFormatVersion = "11" // calculateStableChecksum computes the SBOM artifact cache checksum. Together with the // parent stage digest it forms the cache key: a previously attached SBOM is reused only diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go new file mode 100644 index 0000000000..8063c23136 --- /dev/null +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -0,0 +1,147 @@ +package build + +import ( + "archive/tar" + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "net/http/httptest" + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + + "github.com/werf/werf/v3/pkg/docker_registry" + "github.com/werf/werf/v3/pkg/oci/artifact" +) + +var _ = Describe("recordShippedFiles", func() { + var ( + server *httptest.Server + repo string + ) + + layerOf := func(files, symlinks map[string]string) v1.Layer { + var buf bytes.Buffer + writer := tar.NewWriter(&buf) + for _, path := range lo.Keys(files) { + body := files[path] + Expect(writer.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(path, "/"), + Typeflag: tar.TypeReg, + Size: int64(len(body)), + Mode: 0o644, + })).To(Succeed()) + _, err := io.WriteString(writer, body) + Expect(err).To(Succeed()) + } + for _, path := range lo.Keys(symlinks) { + Expect(writer.WriteHeader(&tar.Header{ + Name: strings.TrimPrefix(path, "/"), + Typeflag: tar.TypeSymlink, + Linkname: symlinks[path], + Mode: 0o777, + })).To(Succeed()) + } + Expect(writer.Close()).To(Succeed()) + + contents := buf.Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(contents)), nil + }) + Expect(err).To(Succeed()) + + return layer + } + + pushImage := func(ctx SpecContext, files, symlinks map[string]string) string { + img, err := mutate.AppendLayers(empty.Image, layerOf(files, symlinks)) + Expect(err).To(Succeed()) + + digest, err := img.Digest() + Expect(err).To(Succeed()) + + ref, err := name.NewDigest(repo + "@" + digest.String()) + Expect(err).To(Succeed()) + Expect(remote.Write(ref, img, remote.WithContext(ctx))).To(Succeed()) + + return digest.String() + } + + digestOf := func(body string) string { + sum := sha256.Sum256([]byte(body)) + + return "sha256:" + hex.EncodeToString(sum[:]) + } + + BeforeEach(func(ctx SpecContext) { + Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + + server = httptest.NewServer(registry.New()) + DeferCleanup(server.Close) + repo = strings.TrimPrefix(server.URL, "http://") + "/test/stages" + }) + + It("records the files of the image's own packages onto its BOM", func(ctx SpecContext) { + digest := pushImage(ctx, map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/var/lib/pm/files.json": `{"zlib":{"version":"1.3.1","files":{"/usr/lib/libz.so.1.3.1":"sha256:recorded-at-unpack"},"symlinks":{"/usr/lib/libz.so.1":"libz.so.1.3.1"}}}`, + "/usr/lib/libz.so.1.3.1": "zlib body", + "/usr/lib/python3/requests-2.32.3.dist-info/RECORD": "requests/__init__.py,sha256=x,12\n", + "/usr/lib/python3/requests/__init__.py": "requests body", + }, map[string]string{ + "/usr/lib/libz.so.1": "libz.so.1.3.1", + }) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1?containerfactoryversion=1"}, + {BOMRef: "requests", Name: "requests", PackageURL: "pkg:pypi/requests@2.32.3"}, + {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, + }} + + Expect(recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest)).To(Succeed()) + + components := *bom.Components + Expect(lo.FromPtr(components[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1", AdditionalContext: "symlink"}, + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("zlib body")}, + })) + Expect(lo.FromPtr(components[1].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("requests body")}, + })) + Expect(components[2].Evidence).To(BeNil()) + }) + + It("leaves os-pm packages unmarked when the image was installed by a pm that wrote no file list", func(ctx SpecContext) { + digest := pushImage(ctx, map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/usr/lib/libz.so.1.3.1": "zlib body", + }, nil) + + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + Expect(recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest)).To(Succeed()) + + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("fails when the built image cannot be read", func(ctx SpecContext) { + bom := &cdx.BOM{Components: &[]cdx.Component{}} + + err := recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + + Expect(err).To(MatchError(ContainSubstring("read built image"))) + }) +}) diff --git a/pkg/oci/artifact/store.go b/pkg/oci/artifact/store.go index 57186d8fb4..dac3559c0d 100644 --- a/pkg/oci/artifact/store.go +++ b/pkg/oci/artifact/store.go @@ -46,6 +46,23 @@ func NewOCIStore(repo, imageName string, opts ...remote.Option) *OCIStore { } } +// ParentImage fetches the image an artifact would be attached to. The digest names one +// manifest, so a multi-platform build gets the image of the platform it is converging, +// not the index. +func (s *OCIStore) ParentImage(ctx context.Context, parentDigest string) (v1.Image, error) { + ref, err := name.NewDigest(s.repo + "@" + parentDigest) + if err != nil { + return nil, fmt.Errorf("parse parent digest reference: %w", err) + } + + img, err := remote.Image(ref, s.remoteOptions(ctx)...) + if err != nil { + return nil, fmt.Errorf("get parent image: %w", err) + } + + return img, nil +} + func (s *OCIStore) Attach(ctx context.Context, parentDigest, artifactType string, payload []byte, checksum, targetPlatform, predicateType string) error { return s.AttachSuperseding(ctx, parentDigest, artifactType, payload, checksum, targetPlatform, predicateType, nil) } diff --git a/pkg/sbom/shippedscope/fileset.go b/pkg/sbom/shippedscope/fileset.go index b49fe0576c..1ba1efaab0 100644 --- a/pkg/sbom/shippedscope/fileset.go +++ b/pkg/sbom/shippedscope/fileset.go @@ -28,6 +28,11 @@ type Fileset struct { Version string Files map[string]File Symlinks map[string]string + // DeclaresFiles tells whether the component named any path at all. One that names none + // — a metapackage, a distribution whose record is empty — can be shown neither present + // nor absent by its files, which is a different thing from one whose declared files are + // all gone. + DeclaresFiles bool } type Filesets struct { diff --git a/pkg/sbom/shippedscope/javascript_fileset.go b/pkg/sbom/shippedscope/javascript_fileset.go index 705c6e2552..1be073c01d 100644 --- a/pkg/sbom/shippedscope/javascript_fileset.go +++ b/pkg/sbom/shippedscope/javascript_fileset.go @@ -44,14 +44,16 @@ func javascriptFilesets(index *FileIndex, manifests map[string][]byte) []Fileset root := path.Dir(manifest) - files, symlinks := collect(index, packageTreePaths(index, root)) + declared := packageTreePaths(index, root) + files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemJavaScript, - Name: declaration.Name, - Version: declaration.Version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemJavaScript, + Name: declaration.Name, + Version: declaration.Version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/lua_fileset.go b/pkg/sbom/shippedscope/lua_fileset.go index 4c8c6e053f..5ec126beea 100644 --- a/pkg/sbom/shippedscope/lua_fileset.go +++ b/pkg/sbom/shippedscope/lua_fileset.go @@ -62,11 +62,12 @@ func luaFilesets(ctx context.Context, index *FileIndex, manifests map[string][]b files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemLua, - Name: name, - Version: version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemLua, + Name: name, + Version: version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/lua_fileset_test.go b/pkg/sbom/shippedscope/lua_fileset_test.go index 7a499cebef..e38bc43e92 100644 --- a/pkg/sbom/shippedscope/lua_fileset_test.go +++ b/pkg/sbom/shippedscope/lua_fileset_test.go @@ -111,4 +111,18 @@ var _ = Describe("CollectFilesets, Lua", func() { Expect(filesets.Filesets).To(BeEmpty()) }) + + It("says a rock whose manifest names no file declares none", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("usr/local/lib/luarocks/rocks-5.4/meta/1.0-1/rock_manifest", "rock_manifest = {\n lua = {\n }\n}\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + meta := filesetOf(filesets, EcosystemLua, "meta", "1.0-1") + Expect(meta).NotTo(BeNil()) + Expect(meta.DeclaresFiles).To(BeFalse()) + }) }) diff --git a/pkg/sbom/shippedscope/occurrences.go b/pkg/sbom/shippedscope/occurrences.go new file mode 100644 index 0000000000..f389778505 --- /dev/null +++ b/pkg/sbom/shippedscope/occurrences.go @@ -0,0 +1,176 @@ +package shippedscope + +import ( + "maps" + "slices" + "strings" + + cdx "github.com/CycloneDX/cyclonedx-go" + packageurl "github.com/package-url/packageurl-go" + "github.com/samber/lo" +) + +// RecordOccurrences writes the files of each fileset onto the component that owns them, as +// evidence.occurrences: the path the file has in this image as location, and what is known +// of its content as additionalContext — the digest of a regular file, or symlinkContext +// for a symlink, which holds no bytes of its own. Every occurrence written here carries +// one or the other, so a reader can tell werf's own evidence from an occurrence that +// arrived in a merged document and means something else entirely. +// +// A component no fileset names is left as it was, and so is one whose filesets all declare +// no files at all. Nothing is then known about its files — its ecosystem records no file +// lists, it was installed in a way none of them reads, or it is a metapackage that ships +// nothing — and an empty occurrence list would instead claim its files were looked for and +// none were found. That is what keeps os-pm components unmarked when the image was +// installed by a pm too old to write a file list. +// +// Occurrences the component already carries are replaced: a fileset is what the image +// holds, and a scanner's guess at where it saw the package is not evidence of the same +// kind. +// +// Returns the number of components given occurrences. +func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int { + if bom == nil || len(filesets) == 0 { + return 0 + } + + owned := make(map[componentKey][]Fileset, len(filesets)) + for _, fileset := range filesets { + key := componentKey{ + ecosystem: fileset.Ecosystem, + name: canonicalName(fileset.Ecosystem, fileset.Name), + version: fileset.Version, + } + owned[key] = append(owned[key], fileset) + } + + recorded := 0 + walkComponents(bom, func(comp *cdx.Component) { + key, ok := keyOfComponent(comp) + if !ok { + return + } + + sets, ok := owned[key] + if !ok || !lo.SomeBy(sets, func(fileset Fileset) bool { return fileset.DeclaresFiles }) { + return + } + + setOccurrences(comp, sets) + recorded++ + }) + + return recorded +} + +// componentKey is what a fileset and a component have to agree on to be the same install: +// one ecosystem's idea of a package, not another's. Two ecosystems naming a package alike +// are two packages. +type componentKey struct { + ecosystem Ecosystem + name string + version string +} + +func keyOfComponent(comp *cdx.Component) (componentKey, bool) { + if comp.PackageURL == "" { + return componentKey{}, false + } + + purl, err := packageurl.FromString(comp.PackageURL) + if err != nil { + return componentKey{}, false + } + + ecosystem, ok := purlEcosystems[purl.Type] + if !ok { + return componentKey{}, false + } + + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + purl.Name + } + + return componentKey{ + ecosystem: ecosystem, + name: canonicalName(ecosystem, name), + version: purl.Version, + }, true +} + +// purlEcosystems maps the purl type of a component to the ecosystem whose filesets can +// describe it. os-pm packages are pkg:generic, the type werf gives what its own pm +// installs; a deb or rpm component comes from a package manager whose file list nothing +// here reads, so it has no fileset and stays unmarked. +var purlEcosystems = map[string]Ecosystem{ + packageurl.TypeGeneric: EcosystemOSPM, + packageurl.TypePyPi: EcosystemPython, + packageurl.TypeNPM: EcosystemJavaScript, + luarocksPurlType: EcosystemLua, +} + +const luarocksPurlType = "luarocks" + +// canonicalName spells a name the way its ecosystem considers canonical, so that a +// fileset and a component that mean the same package are keyed alike: PEP 503 folds +// separator runs and case, npm names are case-insensitive. +func canonicalName(ecosystem Ecosystem, name string) string { + switch ecosystem { + case EcosystemPython: + return normalizePythonName(name) + case EcosystemJavaScript: + return strings.ToLower(name) + default: + return name + } +} + +// symlinkContext marks an occurrence as a symlink, which has a path in the image but no +// content of its own to compare. +const symlinkContext = "symlink" + +// setOccurrences records every file of every fileset of one component. npm installs a +// package more than once under different paths, and each copy is a fileset of its own; +// what the component ships is all of them. +func setOccurrences(comp *cdx.Component, sets []Fileset) { + contexts := make(map[string]string) + for _, fileset := range sets { + for file, entry := range fileset.Files { + contexts[file] = entry.Digest + } + for link := range fileset.Symlinks { + if _, isFile := contexts[link]; !isFile { + contexts[link] = symlinkContext + } + } + } + + occurrences := make([]cdx.EvidenceOccurrence, 0, len(contexts)) + for _, file := range slices.Sorted(maps.Keys(contexts)) { + occurrences = append(occurrences, cdx.EvidenceOccurrence{ + Location: file, + AdditionalContext: contexts[file], + }) + } + + if comp.Evidence == nil { + comp.Evidence = &cdx.Evidence{} + } + comp.Evidence.Occurrences = &occurrences +} + +func walkComponents(bom *cdx.BOM, visit func(comp *cdx.Component)) { + var walk func(components []cdx.Component) + walk = func(components []cdx.Component) { + for i := range components { + visit(&components[i]) + walk(lo.FromPtr(components[i].Components)) + } + } + + walk(lo.FromPtr(bom.Components)) + if bom.Metadata != nil && bom.Metadata.Component != nil { + walk(lo.FromPtr(bom.Metadata.Component.Components)) + } +} diff --git a/pkg/sbom/shippedscope/occurrences_test.go b/pkg/sbom/shippedscope/occurrences_test.go new file mode 100644 index 0000000000..f6fcd14bd3 --- /dev/null +++ b/pkg/sbom/shippedscope/occurrences_test.go @@ -0,0 +1,276 @@ +package shippedscope + +import ( + cdx "github.com/CycloneDX/cyclonedx-go" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" +) + +var _ = Describe("RecordOccurrences", func() { + componentOf := func(purl string) cdx.Component { + return cdx.Component{BOMRef: purl, Name: purl, PackageURL: purl} + } + + bomOf := func(purls ...string) *cdx.BOM { + components := make([]cdx.Component, 0, len(purls)) + for _, purl := range purls { + components = append(components, componentOf(purl)) + } + + return &cdx.BOM{Components: &components} + } + + occurrencesOf := func(bom *cdx.BOM, index int) []cdx.EvidenceOccurrence { + comp := (*bom.Components)[index] + if comp.Evidence == nil { + return nil + } + + return lo.FromPtr(comp.Evidence.Occurrences) + } + + fileset := func(ecosystem Ecosystem, name, version string, files, symlinks map[string]string) Fileset { + entries := make(map[string]File, len(files)) + for file, body := range files { + entries[file] = File{Digest: digestOf(body), Size: int64(len(body))} + } + + return Fileset{ + Ecosystem: ecosystem, + Name: name, + Version: version, + Files: entries, + Symlinks: symlinks, + DeclaresFiles: true, + } + } + + It("records the files of a component of each ecosystem", func() { + bom := bomOf( + "pkg:generic/zlib@1.3.1?containerfactoryversion=1", + "pkg:pypi/requests@2.32.3", + "pkg:npm/debug@4.3.4", + "pkg:luarocks/luasocket@3.1.0-1", + ) + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + fileset(EcosystemPython, "requests", "2.32.3", map[string]string{"/usr/lib/python3/requests/__init__.py": "r"}, nil), + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + fileset(EcosystemLua, "luasocket", "3.1.0-1", map[string]string{"/usr/lib/lua/socket/core.so": "l"}, nil), + }) + + Expect(recorded).To(Equal(4)) + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + Expect(occurrencesOf(bom, 1)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + })) + Expect(occurrencesOf(bom, 2)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/app/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + })) + Expect(occurrencesOf(bom, 3)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/lua/socket/core.so", AdditionalContext: digestOf("l")}, + })) + }) + + It("marks a symlink as one instead of leaving it to be told apart by a missing digest", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", + map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, + map[string]string{"/usr/lib/libz.so.1": "libz.so.1.3.1"}, + ), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1", AdditionalContext: "symlink"}, + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + }) + + It("leaves a component that declares no files alone, instead of saying none were found", func() { + bom := bomOf("pkg:generic/build-essential@12.9") + + recorded := RecordOccurrences(bom, []Fileset{ + {Ecosystem: EcosystemOSPM, Name: "build-essential", Version: "12.9"}, + }) + + Expect(recorded).To(BeZero()) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("records a component one of whose copies declares files", func() { + bom := bomOf("pkg:npm/debug@4.3.4") + + recorded := RecordOccurrences(bom, []Fileset{ + {Ecosystem: EcosystemJavaScript, Name: "debug", Version: "4.3.4"}, + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("leaves a component of another version alone", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.2.13", map[string]string{"/usr/lib/libz.so.1.2.13": "old"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) + + It("leaves a component of another ecosystem alone, however alike it is named", func() { + bom := bomOf("pkg:pypi/debug@4.3.4") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) + + It("matches a PyPI component whose purl keeps a separator the install folds", func() { + bom := bomOf("pkg:pypi/ruamel.yaml@0.18.6") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemPython, "ruamel-yaml", "0.18.6", map[string]string{"/usr/lib/python3/ruamel/yaml/__init__.py": "y"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("matches an npm component whose package.json spells the name in mixed case", func() { + bom := bomOf("pkg:npm/JSONStream@1.3.5") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "JSONStream", "1.3.5", map[string]string{"/app/node_modules/JSONStream/index.js": "j"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("matches a scoped npm component to the package tree it was installed into", func() { + bom := bomOf("pkg:npm/%40babel/core@7.24.0") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "@babel/core", "7.24.0", map[string]string{"/app/node_modules/@babel/core/index.js": "b"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("gives a component installed more than once the files of every copy", func() { + bom := bomOf("pkg:npm/debug@4.3.4") + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/debug/index.js": "d"}, nil), + fileset(EcosystemJavaScript, "debug", "4.3.4", map[string]string{"/app/node_modules/ms/node_modules/debug/index.js": "d"}, nil), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/app/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + {Location: "/app/node_modules/ms/node_modules/debug/index.js", AdditionalContext: digestOf("d")}, + })) + }) + + It("says the files were looked for when a fileset names a component but the image kept none", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", nil, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect((*bom.Components)[0].Evidence.Occurrences).To(Equal(&[]cdx.EvidenceOccurrence{})) + }) + + It("leaves a component no fileset names unmarked, so nothing is known of its files", func() { + bom := bomOf("pkg:deb/debian/libc6@2.36", "pkg:golang/gopkg.in/yaml.v3@3.0.1") + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "libc6", "2.36", map[string]string{"/lib/libc.so.6": "c"}, nil), + }) + + Expect(recorded).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + Expect(occurrencesOf(bom, 1)).To(BeNil()) + }) + + It("records a component nested under another", func() { + nested := []cdx.Component{componentOf("pkg:generic/zlib@1.3.1")} + outer := componentOf("pkg:generic/git@2.50.1") + outer.Components = &nested + bom := &cdx.BOM{Components: &[]cdx.Component{outer}} + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(lo.FromPtr((*(*bom.Components)[0].Components)[0].Evidence.Occurrences)).To(HaveLen(1)) + }) + + It("records a component listed under the image's own component", func() { + nested := []cdx.Component{componentOf("pkg:generic/zlib@1.3.1")} + bom := &cdx.BOM{Metadata: &cdx.Metadata{Component: &cdx.Component{ + Type: cdx.ComponentTypeContainer, + Components: &nested, + }}} + + recorded := RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(recorded).To(Equal(1)) + Expect(lo.FromPtr(nested[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/libz.so.1.3.1", AdditionalContext: digestOf("z")}, + })) + }) + + It("replaces the occurrences a scanner guessed with the files the image holds", func() { + bom := bomOf("pkg:pypi/requests@2.32.3") + (*bom.Components)[0].Evidence = &cdx.Evidence{ + Occurrences: &[]cdx.EvidenceOccurrence{{Location: "/usr/lib/python3/requests-2.32.3.dist-info/RECORD"}}, + } + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemPython, "requests", "2.32.3", map[string]string{"/usr/lib/python3/requests/__init__.py": "r"}, nil), + }) + + Expect(occurrencesOf(bom, 0)).To(Equal([]cdx.EvidenceOccurrence{ + {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("r")}, + })) + }) + + It("keeps the evidence a component already carries of another kind", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + (*bom.Components)[0].Evidence = &cdx.Evidence{ + Identity: &[]cdx.EvidenceIdentity{{Field: cdx.EvidenceIdentityFieldTypeCPE}}, + } + + RecordOccurrences(bom, []Fileset{ + fileset(EcosystemOSPM, "zlib", "1.3.1", map[string]string{"/usr/lib/libz.so.1.3.1": "z"}, nil), + }) + + Expect(lo.FromPtr((*bom.Components)[0].Evidence.Identity)).To(HaveLen(1)) + Expect(occurrencesOf(bom, 0)).To(HaveLen(1)) + }) + + It("does nothing to a BOM when the image yielded no filesets", func() { + bom := bomOf("pkg:generic/zlib@1.3.1") + + Expect(RecordOccurrences(bom, nil)).To(BeZero()) + Expect(occurrencesOf(bom, 0)).To(BeNil()) + }) +}) diff --git a/pkg/sbom/shippedscope/os_pm_fileset.go b/pkg/sbom/shippedscope/os_pm_fileset.go index f97545ac43..f3a18b5121 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset.go +++ b/pkg/sbom/shippedscope/os_pm_fileset.go @@ -62,11 +62,12 @@ func osPMFilesets(ctx context.Context, index *FileIndex, manifests map[string][] files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemOSPM, - Name: name, - Version: entry.Version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemOSPM, + Name: name, + Version: entry.Version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/os_pm_fileset_test.go b/pkg/sbom/shippedscope/os_pm_fileset_test.go index 3068cc3bd8..55d5e075a5 100644 --- a/pkg/sbom/shippedscope/os_pm_fileset_test.go +++ b/pkg/sbom/shippedscope/os_pm_fileset_test.go @@ -65,6 +65,21 @@ var _ = Describe("CollectFilesets, os-pm", func() { git := filesetOf(filesets, EcosystemOSPM, "git", "2.50.1") Expect(git).NotTo(BeNil()) Expect(git.Files).To(HaveLen(1)) + Expect(git.DeclaresFiles).To(BeTrue()) + }) + + It("tells a package that recorded no files from one whose recorded files are gone", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("var/lib/pm/files.json", + `{"build-essential": {"version": "12.9"}, "git": {"version": "2.50.1", "files": {"/usr/bin/git": "sha256:dead"}}}`), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + Expect(filesetOf(filesets, EcosystemOSPM, "build-essential", "12.9").DeclaresFiles).To(BeFalse()) + Expect(filesetOf(filesets, EcosystemOSPM, "git", "2.50.1").DeclaresFiles).To(BeTrue()) }) It("reads a file list the image reaches through a hardlink", func(ctx SpecContext) { diff --git a/pkg/sbom/shippedscope/python_fileset.go b/pkg/sbom/shippedscope/python_fileset.go index a93f11c3b2..f01dda37b8 100644 --- a/pkg/sbom/shippedscope/python_fileset.go +++ b/pkg/sbom/shippedscope/python_fileset.go @@ -50,11 +50,12 @@ func pythonFilesets(ctx context.Context, index *FileIndex, manifests map[string] files, symlinks := collect(index, declared) filesets = append(filesets, Fileset{ - Ecosystem: EcosystemPython, - Name: normalizePythonName(name), - Version: version, - Files: files, - Symlinks: symlinks, + Ecosystem: EcosystemPython, + Name: normalizePythonName(name), + Version: version, + Files: files, + Symlinks: symlinks, + DeclaresFiles: len(declared) > 0, }) } diff --git a/pkg/sbom/shippedscope/python_fileset_test.go b/pkg/sbom/shippedscope/python_fileset_test.go index 30053a9ab0..f0f5ae7ace 100644 --- a/pkg/sbom/shippedscope/python_fileset_test.go +++ b/pkg/sbom/shippedscope/python_fileset_test.go @@ -123,6 +123,22 @@ var _ = Describe("CollectFilesets, Python", func() { stub := filesetOf(filesets, EcosystemPython, "stub", "1.0") Expect(stub).NotTo(BeNil()) Expect(stub.Files).To(BeEmpty()) + Expect(stub.DeclaresFiles).To(BeFalse()) + }) + + It("says a distribution declares files when its record names some, however few survived", func(ctx SpecContext) { + img, err := imageOf([]tarEntry{ + regularFile("site/gone-1.0.dist-info/RECORD", "gone/__init__.py,sha256=x,12\n"), + }) + Expect(err).NotTo(HaveOccurred()) + + filesets, err := CollectFilesets(ctx, img) + Expect(err).NotTo(HaveOccurred()) + + gone := filesetOf(filesets, EcosystemPython, "gone", "1.0") + Expect(gone).NotTo(BeNil()) + Expect(gone.Files).To(BeEmpty()) + Expect(gone.DeclaresFiles).To(BeTrue()) }) It("reads a record whose path is quoted because it holds a comma", func(ctx SpecContext) { From e8bb5cfb70a3483cfe655a13e9e7f0e4b3575409 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 12:31:05 +0300 Subject: [PATCH 2/6] fix(sbom): keep a build that cannot read its own image, and skip the read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recording which files a component consists of is an enrichment: an SBOM without those records is the SBOM werf wrote before it kept them, and a reader that finds no record of a component's files keeps the component. A files.json that does not parse, a layer whose digest does not verify, a registry that times out on the second read — each of those failed a build that succeeded the day before, and failing bought nothing. They now warn and leave the BOM as it was. The image is read only when the BOM holds a component some ecosystem could record files for. An image whose components are all Go modules or deb packages — a scratch Go image, a plain Dockerfile build — paid a full download of its layers for nothing. Also from review: the guard that let a regular file keep its digest against a symlink of the same path in another fileset of the same component is gone, since an index holds a path as one or the other and the filesets of one component come from one index; and the godoc says why taking pkg:generic for os-pm is safe when syft gives that type to runtimes too. Signed-off-by: Radmir Khurum --- pkg/build/sbom_step.go | 25 ++++++---- pkg/build/sbom_step_shipped_files_test.go | 57 ++++++++++++++++++++--- pkg/sbom/shippedscope/occurrences.go | 27 +++++++++-- pkg/sbom/shippedscope/occurrences_test.go | 37 +++++++++++++++ 4 files changed, 129 insertions(+), 17 deletions(-) diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index efb081830a..7b61e57248 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -203,9 +203,7 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) - if err := recordShippedFiles(ctx, resultBOM, store, parentDigest); err != nil { - return err - } + recordShippedFiles(ctx, resultBOM, store, parentDigest) resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { @@ -234,21 +232,32 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, // Runs last, once the BOM holds the components it will ship with: Canonicalize folds // duplicate components together and drops an evidence list that comes out empty, which is // exactly the "its files were looked for and none were found" this writes. -func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, parentDigest string) error { +// +// Nothing here fails the build. A BOM without these records is the BOM werf wrote before +// it kept them, and a reader that finds no record of a component's files keeps the +// component, so the worst an unreadable image costs is the SBOM of the image built from +// this one being as complete as it used to be. +func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, parentDigest string) { + if !shippedscope.HasFileBackedComponents(bom) { + return + } + img, err := store.ParentImage(ctx, parentDigest) if err != nil { - return fmt.Errorf("read built image: %w", err) + logboek.Context(ctx).Warn().LogF("WARNING: unable to read the built image; the SBOM records none of the files its components consist of: %s\n", err) + + return } filesets, err := shippedscope.CollectFilesets(ctx, img) if err != nil { - return fmt.Errorf("collect component filesets: %w", err) + logboek.Context(ctx).Warn().LogF("WARNING: unable to read which files the components of the image consist of; the SBOM records none of them: %s\n", err) + + return } recorded := shippedscope.RecordOccurrences(bom, filesets.Filesets) logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) - - return nil } // restoreImageMetadata sets the BOM's top-level component to the scanned image while diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go index 8063c23136..092763ed88 100644 --- a/pkg/build/sbom_step_shipped_files_test.go +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -3,6 +3,7 @@ package build import ( "archive/tar" "bytes" + "context" "crypto/sha256" "encoding/hex" "io" @@ -21,6 +22,7 @@ import ( . "github.com/onsi/gomega" "github.com/samber/lo" + "github.com/werf/logboek" "github.com/werf/werf/v3/pkg/docker_registry" "github.com/werf/werf/v3/pkg/oci/artifact" ) @@ -84,6 +86,13 @@ var _ = Describe("recordShippedFiles", func() { return "sha256:" + hex.EncodeToString(sum[:]) } + logged := func(ctx SpecContext, record func(ctx context.Context)) string { + var out bytes.Buffer + record(logboek.NewContext(ctx, logboek.NewLogger(&out, &out))) + + return out.String() + } + BeforeEach(func(ctx SpecContext) { Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) @@ -109,7 +118,7 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} - Expect(recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest)).To(Succeed()) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) components := *bom.Components Expect(lo.FromPtr(components[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ @@ -132,16 +141,52 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, }} - Expect(recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest)).To(Succeed()) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) Expect((*bom.Components)[0].Evidence).To(BeNil()) }) - It("fails when the built image cannot be read", func(ctx SpecContext) { - bom := &cdx.BOM{Components: &[]cdx.Component{}} + It("warns and leaves the BOM as it was when the built image cannot be read", func(ctx SpecContext) { + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + }) + + Expect(out).To(ContainSubstring("unable to read the built image")) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("warns and leaves the BOM as it was when the image's file lists cannot be read", func(ctx SpecContext) { + digest := pushImage(ctx, map[string]string{ + "/var/lib/pm/index.json": `{}`, + "/var/lib/pm/files.json": `not json at all`, + }, nil) - err := recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) + }) + + Expect(out).To(ContainSubstring("unable to read which files the components")) + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("does not read the image when no component could be given files", func(ctx SpecContext) { + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "yaml", Name: "gopkg.in/yaml.v3", PackageURL: "pkg:golang/gopkg.in/yaml.v3@3.0.1"}, + {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + }) - Expect(err).To(MatchError(ContainSubstring("read built image"))) + Expect(out).To(BeEmpty()) }) }) diff --git a/pkg/sbom/shippedscope/occurrences.go b/pkg/sbom/shippedscope/occurrences.go index f389778505..dd2ae8a17e 100644 --- a/pkg/sbom/shippedscope/occurrences.go +++ b/pkg/sbom/shippedscope/occurrences.go @@ -63,6 +63,25 @@ func RecordOccurrences(bom *cdx.BOM, filesets []Fileset) int { return recorded } +// HasFileBackedComponents tells whether the BOM holds a component any ecosystem could +// record files for. Reading an image to find out costs a full download of its layers, and +// an image whose components are all Go modules, deb packages or anything else no collector +// reads has nothing to gain from it. +func HasFileBackedComponents(bom *cdx.BOM) bool { + if bom == nil { + return false + } + + found := false + walkComponents(bom, func(comp *cdx.Component) { + if _, ok := keyOfComponent(comp); ok { + found = true + } + }) + + return found +} + // componentKey is what a fileset and a component have to agree on to be the same install: // one ecosystem's idea of a package, not another's. Two ecosystems naming a package alike // are two packages. @@ -103,6 +122,10 @@ func keyOfComponent(comp *cdx.Component) (componentKey, bool) { // describe it. os-pm packages are pkg:generic, the type werf gives what its own pm // installs; a deb or rpm component comes from a package manager whose file list nothing // here reads, so it has no fileset and stays unmarked. +// +// pkg:generic is not werf's alone — syft gives it to a runtime it recognizes by its binary +// — but taking a fileset needs the same name and version as a package the pm recorded, and +// a component agreeing with a pm entry on both is that package. var purlEcosystems = map[string]Ecosystem{ packageurl.TypeGeneric: EcosystemOSPM, packageurl.TypePyPi: EcosystemPython, @@ -140,9 +163,7 @@ func setOccurrences(comp *cdx.Component, sets []Fileset) { contexts[file] = entry.Digest } for link := range fileset.Symlinks { - if _, isFile := contexts[link]; !isFile { - contexts[link] = symlinkContext - } + contexts[link] = symlinkContext } } diff --git a/pkg/sbom/shippedscope/occurrences_test.go b/pkg/sbom/shippedscope/occurrences_test.go index f6fcd14bd3..8abf47d3d8 100644 --- a/pkg/sbom/shippedscope/occurrences_test.go +++ b/pkg/sbom/shippedscope/occurrences_test.go @@ -274,3 +274,40 @@ var _ = Describe("RecordOccurrences", func() { Expect(occurrencesOf(bom, 0)).To(BeNil()) }) }) + +var _ = Describe("HasFileBackedComponents", func() { + bomOf := func(purls ...string) *cdx.BOM { + components := make([]cdx.Component, 0, len(purls)) + for _, purl := range purls { + components = append(components, cdx.Component{BOMRef: purl, Name: purl, PackageURL: purl}) + } + + return &cdx.BOM{Components: &components} + } + + It("finds a component of each ecosystem that records file lists", func() { + Expect(HasFileBackedComponents(bomOf("pkg:generic/zlib@1.3.1"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:pypi/requests@2.32.3"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:npm/debug@4.3.4"))).To(BeTrue()) + Expect(HasFileBackedComponents(bomOf("pkg:luarocks/luasocket@3.1.0-1"))).To(BeTrue()) + }) + + It("finds nothing in a BOM whose components no ecosystem here records", func() { + Expect(HasFileBackedComponents(bomOf( + "pkg:golang/gopkg.in/yaml.v3@3.0.1", + "pkg:deb/debian/libc6@2.36", + "pkg:cargo/serde@1.0.0", + ))).To(BeFalse()) + }) + + It("finds a component nested under another", func() { + nested := []cdx.Component{{BOMRef: "z", PackageURL: "pkg:generic/zlib@1.3.1"}} + outer := cdx.Component{BOMRef: "g", PackageURL: "pkg:golang/x@1", Components: &nested} + + Expect(HasFileBackedComponents(&cdx.BOM{Components: &[]cdx.Component{outer}})).To(BeTrue()) + }) + + It("finds nothing in an empty BOM", func() { + Expect(HasFileBackedComponents(&cdx.BOM{})).To(BeFalse()) + }) +}) From cec7a642bcf530288724eacc95166d82bda94835 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 13:46:00 +0300 Subject: [PATCH 3/6] fix(sbom): say in the warning that the degraded SBOM is cached Found by the second independent review of the fail-open: the SBOM cache is keyed on the stable checksum, which does not encode whether the files of the components were recorded. A transient registry failure on the second read therefore warned once and cached an SBOM without these records under the same checksum, and no later build of that stage retried. The warning now names the image and its digest, so it can be found in a flat CI log, and says the SBOM is cached as such until the stage is rebuilt. Signed-off-by: Radmir Khurum --- pkg/build/sbom_step.go | 12 +++++++----- pkg/build/sbom_step_shipped_files_test.go | 22 +++++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 7b61e57248..030cfcf1f7 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -203,7 +203,7 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) - recordShippedFiles(ctx, resultBOM, store, parentDigest) + recordShippedFiles(ctx, resultBOM, store, werfImgName, parentDigest) resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { @@ -236,22 +236,24 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, // Nothing here fails the build. A BOM without these records is the BOM werf wrote before // it kept them, and a reader that finds no record of a component's files keeps the // component, so the worst an unreadable image costs is the SBOM of the image built from -// this one being as complete as it used to be. -func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, parentDigest string) { +// this one being as complete as it used to be. That BOM is then cached against the stage +// like any other, so the warning says so: a transient failure is not retried on the next +// build of the same stage. +func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, werfImgName, parentDigest string) { if !shippedscope.HasFileBackedComponents(bom) { return } img, err := store.ParentImage(ctx, parentDigest) if err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: unable to read the built image; the SBOM records none of the files its components consist of: %s\n", err) + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read the built image %s; the SBOM records none of the files its components consist of and is cached as such until the stage is rebuilt: %s\n", werfImgName, parentDigest, err) return } filesets, err := shippedscope.CollectFilesets(ctx, img) if err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: unable to read which files the components of the image consist of; the SBOM records none of them: %s\n", err) + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read which files the components of the built image %s consist of; the SBOM records none of them and is cached as such until the stage is rebuilt: %s\n", werfImgName, parentDigest, err) return } diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go index 092763ed88..8543cf97f6 100644 --- a/pkg/build/sbom_step_shipped_files_test.go +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -88,7 +88,9 @@ var _ = Describe("recordShippedFiles", func() { logged := func(ctx SpecContext, record func(ctx context.Context)) string { var out bytes.Buffer - record(logboek.NewContext(ctx, logboek.NewLogger(&out, &out))) + logger := logboek.NewLogger(&out, &out) + logger.Streams().DisableLineWrapping() + record(logboek.NewContext(ctx, logger)) return out.String() } @@ -118,7 +120,7 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) components := *bom.Components Expect(lo.FromPtr(components[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ @@ -141,7 +143,7 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, }} - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) Expect((*bom.Components)[0].Evidence).To(BeNil()) }) @@ -152,10 +154,11 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", "sha256:"+strings.Repeat("0", 64)) }) - Expect(out).To(ContainSubstring("unable to read the built image")) + Expect(out).To(ContainSubstring("image app: unable to read the built image sha256:0000")) + Expect(out).To(ContainSubstring("cached as such until the stage is rebuilt")) Expect((*bom.Components)[0].Evidence).To(BeNil()) }) @@ -170,21 +173,22 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), digest) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) }) - Expect(out).To(ContainSubstring("unable to read which files the components")) + Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image " + digest)) + Expect(out).To(ContainSubstring("cached as such until the stage is rebuilt")) Expect((*bom.Components)[0].Evidence).To(BeNil()) }) - It("does not read the image when no component could be given files", func(ctx SpecContext) { + It("warns nothing and skips the read when no component could be given files", func(ctx SpecContext) { bom := &cdx.BOM{Components: &[]cdx.Component{ {BOMRef: "yaml", Name: "gopkg.in/yaml.v3", PackageURL: "pkg:golang/gopkg.in/yaml.v3@3.0.1"}, {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "sha256:"+strings.Repeat("0", 64)) + recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", "sha256:"+strings.Repeat("0", 64)) }) Expect(out).To(BeEmpty()) From 5aa1c59de0c4ff6cab609a6764853d80ea103943 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 14:29:43 +0300 Subject: [PATCH 4/6] fix(sbom): read the built image out of the backend, not the registry again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The step pulls the image into the backend before it does anything else, so reading it from the registry a second time to learn which files its components consist of was a full download of every layer for nothing: 65 s on node:20-slim, close to ten minutes on the 1 GB node:20, on every build of every image with a pip, npm, os-pm or luarocks component. It was also the only reason a degraded SBOM could be cached by accident. A registry read fails transiently; the SBOM written without these records then sat under the same checksum and no later build of that stage retried. A save out of the backend fails only for reasons the next build hits again — an image that is not there, a files.json that does not parse — and those are right to cache. The warning that said the SBOM is cached as such goes with it. SaveImageToStream is on both backends; the tarball goes to a file under the werf tmp dir because tarball.Image reads it once for the manifest and once per layer, the same way MutateAndPushImage does. OCIStore.ParentImage loses its only caller and is gone. Signed-off-by: Radmir Khurum --- pkg/build/sbom_step.go | 67 +++++++++-- pkg/build/sbom_step_shipped_files_test.go | 129 +++++++++++++++------- pkg/oci/artifact/store.go | 17 --- 3 files changed, 148 insertions(+), 65 deletions(-) diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 030cfcf1f7..4eedd40194 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "maps" "os" "path/filepath" @@ -13,6 +14,8 @@ import ( "time" cdx "github.com/CycloneDX/cyclonedx-go" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" packageurl "github.com/package-url/packageurl-go" "github.com/samber/lo" "github.com/sigstore/sigstore/pkg/signature" @@ -34,6 +37,7 @@ import ( "github.com/werf/werf/v3/pkg/sbom/scanner" "github.com/werf/werf/v3/pkg/sbom/shippedscope" "github.com/werf/werf/v3/pkg/storage" + "github.com/werf/werf/v3/pkg/tmp_manager" "github.com/werf/werf/v3/pkg/werf" "github.com/werf/werf/v3/pkg/werf/global_warnings" ) @@ -203,7 +207,7 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, cyclonedxutil.Canonicalize(ctx, resultBOM) - recordShippedFiles(ctx, resultBOM, store, werfImgName, parentDigest) + recordShippedFiles(ctx, resultBOM, step.containerBackend, werfImgName, stageDesc.Info.Name) resultJSON, err := cyclonedxutil.ToJSON(resultBOM) if err != nil { @@ -233,27 +237,30 @@ func (step *sbomStep) ConvergeWithMerge(ctx context.Context, werfImgName string, // duplicate components together and drops an evidence list that comes out empty, which is // exactly the "its files were looked for and none were found" this writes. // +// The image is read out of the backend the step pulled it into, not from the registry a +// second time: the read is then as long as a save to local disk, and what can fail it is +// the host — an image that is not there, a disk that is full — rather than the network. +// // Nothing here fails the build. A BOM without these records is the BOM werf wrote before // it kept them, and a reader that finds no record of a component's files keeps the // component, so the worst an unreadable image costs is the SBOM of the image built from -// this one being as complete as it used to be. That BOM is then cached against the stage -// like any other, so the warning says so: a transient failure is not retried on the next -// build of the same stage. -func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCIStore, werfImgName, parentDigest string) { +// this one being as complete as it used to be. +func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, werfImgName, imageName string) { if !shippedscope.HasFileBackedComponents(bom) { return } - img, err := store.ParentImage(ctx, parentDigest) + img, cleanup, err := savedImage(ctx, backend, imageName) if err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read the built image %s; the SBOM records none of the files its components consist of and is cached as such until the stage is rebuilt: %s\n", werfImgName, parentDigest, err) + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read the built image %s; the SBOM records none of the files its components consist of: %s\n", werfImgName, imageName, err) return } + defer cleanup() filesets, err := shippedscope.CollectFilesets(ctx, img) if err != nil { - logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read which files the components of the built image %s consist of; the SBOM records none of them and is cached as such until the stage is rebuilt: %s\n", werfImgName, parentDigest, err) + logboek.Context(ctx).Warn().LogF("WARNING: image %s: unable to read which files the components of the built image %s consist of; the SBOM records none of them: %s\n", werfImgName, imageName, err) return } @@ -262,6 +269,50 @@ func recordShippedFiles(ctx context.Context, bom *cdx.BOM, store *artifact.OCISt logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) } +type imageSaver interface { + SaveImageToStream(ctx context.Context, imageName string) (io.ReadCloser, error) +} + +// savedImage reads the image out of the backend as a docker-save tarball. The tarball is +// read more than once, once for its manifest and once per layer, which a stream cannot +// give, so it goes to a file first, the same way MutateAndPushImage does. The returned +// cleanup removes that file. +func savedImage(ctx context.Context, backend imageSaver, imageName string) (v1.Image, func(), error) { + stream, err := backend.SaveImageToStream(ctx, imageName) + if err != nil { + return nil, nil, fmt.Errorf("save image: %w", err) + } + defer stream.Close() + + file, err := tmp_manager.TempFile("werf-sbom-shipped-files-*.tar") + if err != nil { + return nil, nil, fmt.Errorf("create temp file: %w", err) + } + cleanup := func() { _ = os.Remove(file.Name()) } + + if _, err := io.Copy(file, stream); err != nil { + _ = file.Close() + cleanup() + + return nil, nil, fmt.Errorf("write saved image to %s: %w", file.Name(), err) + } + + if err := file.Close(); err != nil { + cleanup() + + return nil, nil, fmt.Errorf("close %s: %w", file.Name(), err) + } + + img, err := tarball.ImageFromPath(file.Name(), nil) + if err != nil { + cleanup() + + return nil, nil, fmt.Errorf("read saved image %s: %w", file.Name(), err) + } + + return img, cleanup, nil +} + // restoreImageMetadata sets the BOM's top-level component to the scanned image while // keeping any syft-provided metadata (tools, timestamp). A directory source reports the // temporary scan directory as its component, so it must be replaced; the dependency edges diff --git a/pkg/build/sbom_step_shipped_files_test.go b/pkg/build/sbom_step_shipped_files_test.go index 8543cf97f6..c92b9d3b82 100644 --- a/pkg/build/sbom_step_shipped_files_test.go +++ b/pkg/build/sbom_step_shipped_files_test.go @@ -6,34 +6,46 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "io" - "net/http/httptest" + "path/filepath" "strings" cdx "github.com/CycloneDX/cyclonedx-go" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/registry" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/static" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" "github.com/werf/logboek" - "github.com/werf/werf/v3/pkg/docker_registry" - "github.com/werf/werf/v3/pkg/oci/artifact" + "github.com/werf/werf/v3/pkg/werf" ) -var _ = Describe("recordShippedFiles", func() { - var ( - server *httptest.Server - repo string - ) +type savedImageBackend struct { + tarball []byte + err error + saved []string +} - layerOf := func(files, symlinks map[string]string) v1.Layer { +var _ imageSaver = (*savedImageBackend)(nil) + +func (b *savedImageBackend) SaveImageToStream(_ context.Context, imageName string) (io.ReadCloser, error) { + b.saved = append(b.saved, imageName) + if b.err != nil { + return nil, b.err + } + + return io.NopCloser(bytes.NewReader(b.tarball)), nil +} + +var _ = Describe("recordShippedFiles", func() { + tarOf := func(files, symlinks map[string]string) []byte { var buf bytes.Buffer writer := tar.NewWriter(&buf) for _, path := range lo.Keys(files) { @@ -57,7 +69,12 @@ var _ = Describe("recordShippedFiles", func() { } Expect(writer.Close()).To(Succeed()) - contents := buf.Bytes() + return buf.Bytes() + } + + // Buildah saves layers uncompressed and Docker may save them either way; tarball.Write + // copies what Compressed() gives, so the layer type decides which the saved tarball holds. + gzippedLayer := func(contents []byte) v1.Layer { layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(contents)), nil }) @@ -65,19 +82,25 @@ var _ = Describe("recordShippedFiles", func() { return layer } + uncompressedLayer := func(contents []byte) v1.Layer { + return static.NewLayer(contents, types.DockerUncompressedLayer) + } - pushImage := func(ctx SpecContext, files, symlinks map[string]string) string { - img, err := mutate.AppendLayers(empty.Image, layerOf(files, symlinks)) + backendSaving := func(layerOf func([]byte) v1.Layer, files, symlinks map[string]string) *savedImageBackend { + img, err := mutate.AppendLayers(empty.Image, layerOf(tarOf(files, symlinks))) Expect(err).To(Succeed()) - digest, err := img.Digest() + tag, err := name.NewTag("werf-stages:app") Expect(err).To(Succeed()) - ref, err := name.NewDigest(repo + "@" + digest.String()) - Expect(err).To(Succeed()) - Expect(remote.Write(ref, img, remote.WithContext(ctx))).To(Succeed()) + var saved bytes.Buffer + Expect(tarball.Write(tag, img, &saved)).To(Succeed()) - return digest.String() + return &savedImageBackend{tarball: saved.Bytes()} + } + + backendWith := func(files, symlinks map[string]string) *savedImageBackend { + return backendSaving(gzippedLayer, files, symlinks) } digestOf := func(body string) string { @@ -95,16 +118,19 @@ var _ = Describe("recordShippedFiles", func() { return out.String() } - BeforeEach(func(ctx SpecContext) { - Expect(docker_registry.Init(ctx, false, false, nil, nil)).To(Succeed()) + savedTarballs := func() []string { + matches, err := filepath.Glob(filepath.Join(werf.GetTmpDir(), "werf-sbom-shipped-files-*.tar")) + Expect(err).To(Succeed()) + + return matches + } - server = httptest.NewServer(registry.New()) - DeferCleanup(server.Close) - repo = strings.TrimPrefix(server.URL, "http://") + "/test/stages" + BeforeEach(func() { + Expect(werf.Init(GinkgoT().TempDir(), "")).To(Succeed()) }) - It("records the files of the image's own packages onto its BOM", func(ctx SpecContext) { - digest := pushImage(ctx, map[string]string{ + DescribeTable("records the files of the image's own packages onto its BOM", func(ctx SpecContext, layerOf func([]byte) v1.Layer) { + backend := backendSaving(layerOf, map[string]string{ "/var/lib/pm/index.json": `{}`, "/var/lib/pm/files.json": `{"zlib":{"version":"1.3.1","files":{"/usr/lib/libz.so.1.3.1":"sha256:recorded-at-unpack"},"symlinks":{"/usr/lib/libz.so.1":"libz.so.1.3.1"}}}`, "/usr/lib/libz.so.1.3.1": "zlib body", @@ -120,8 +146,9 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + Expect(backend.saved).To(Equal([]string{"werf-stages:app"})) components := *bom.Components Expect(lo.FromPtr(components[0].Evidence.Occurrences)).To(Equal([]cdx.EvidenceOccurrence{ {Location: "/usr/lib/libz.so.1", AdditionalContext: "symlink"}, @@ -131,10 +158,14 @@ var _ = Describe("recordShippedFiles", func() { {Location: "/usr/lib/python3/requests/__init__.py", AdditionalContext: digestOf("requests body")}, })) Expect(components[2].Evidence).To(BeNil()) - }) + Expect(savedTarballs()).To(BeEmpty()) + }, + Entry("saved with gzipped layers", gzippedLayer), + Entry("saved with uncompressed layers, as Buildah saves them", uncompressedLayer), + ) It("leaves os-pm packages unmarked when the image was installed by a pm that wrote no file list", func(ctx SpecContext) { - digest := pushImage(ctx, map[string]string{ + backend := backendWith(map[string]string{ "/var/lib/pm/index.json": `{}`, "/usr/lib/libz.so.1.3.1": "zlib body", }, nil) @@ -143,27 +174,43 @@ var _ = Describe("recordShippedFiles", func() { {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, }} - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + + Expect((*bom.Components)[0].Evidence).To(BeNil()) + }) + + It("warns and leaves the BOM as it was when the backend cannot save the image", func(ctx SpecContext) { + backend := &savedImageBackend{err: errors.New("no such image")} + bom := &cdx.BOM{Components: &[]cdx.Component{ + {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, + }} + + out := logged(ctx, func(ctx context.Context) { + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") + }) + Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) + Expect(out).To(ContainSubstring("no such image")) Expect((*bom.Components)[0].Evidence).To(BeNil()) }) - It("warns and leaves the BOM as it was when the built image cannot be read", func(ctx SpecContext) { + It("warns and leaves the BOM as it was when the saved image is not an image", func(ctx SpecContext) { + backend := &savedImageBackend{tarball: []byte("not a tarball")} bom := &cdx.BOM{Components: &[]cdx.Component{ {BOMRef: "zlib", Name: "zlib", PackageURL: "pkg:generic/zlib@1.3.1"}, }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", "sha256:"+strings.Repeat("0", 64)) + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") }) - Expect(out).To(ContainSubstring("image app: unable to read the built image sha256:0000")) - Expect(out).To(ContainSubstring("cached as such until the stage is rebuilt")) + Expect(out).To(ContainSubstring("image app: unable to read the built image werf-stages:app")) Expect((*bom.Components)[0].Evidence).To(BeNil()) + Expect(savedTarballs()).To(BeEmpty()) }) It("warns and leaves the BOM as it was when the image's file lists cannot be read", func(ctx SpecContext) { - digest := pushImage(ctx, map[string]string{ + backend := backendWith(map[string]string{ "/var/lib/pm/index.json": `{}`, "/var/lib/pm/files.json": `not json at all`, }, nil) @@ -173,24 +220,26 @@ var _ = Describe("recordShippedFiles", func() { }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", digest) + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") }) - Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image " + digest)) - Expect(out).To(ContainSubstring("cached as such until the stage is rebuilt")) + Expect(out).To(ContainSubstring("image app: unable to read which files the components of the built image werf-stages:app consist of")) Expect((*bom.Components)[0].Evidence).To(BeNil()) + Expect(savedTarballs()).To(BeEmpty()) }) - It("warns nothing and skips the read when no component could be given files", func(ctx SpecContext) { + It("warns nothing and does not save the image when no component could be given files", func(ctx SpecContext) { + backend := &savedImageBackend{err: errors.New("must not be called")} bom := &cdx.BOM{Components: &[]cdx.Component{ {BOMRef: "yaml", Name: "gopkg.in/yaml.v3", PackageURL: "pkg:golang/gopkg.in/yaml.v3@3.0.1"}, {BOMRef: "libc6", Name: "libc6", PackageURL: "pkg:deb/debian/libc6@2.36"}, }} out := logged(ctx, func(ctx context.Context) { - recordShippedFiles(ctx, bom, artifact.NewOCIStore(repo, "app"), "app", "sha256:"+strings.Repeat("0", 64)) + recordShippedFiles(ctx, bom, backend, "app", "werf-stages:app") }) Expect(out).To(BeEmpty()) + Expect(backend.saved).To(BeEmpty()) }) }) diff --git a/pkg/oci/artifact/store.go b/pkg/oci/artifact/store.go index dac3559c0d..57186d8fb4 100644 --- a/pkg/oci/artifact/store.go +++ b/pkg/oci/artifact/store.go @@ -46,23 +46,6 @@ func NewOCIStore(repo, imageName string, opts ...remote.Option) *OCIStore { } } -// ParentImage fetches the image an artifact would be attached to. The digest names one -// manifest, so a multi-platform build gets the image of the platform it is converging, -// not the index. -func (s *OCIStore) ParentImage(ctx context.Context, parentDigest string) (v1.Image, error) { - ref, err := name.NewDigest(s.repo + "@" + parentDigest) - if err != nil { - return nil, fmt.Errorf("parse parent digest reference: %w", err) - } - - img, err := remote.Image(ref, s.remoteOptions(ctx)...) - if err != nil { - return nil, fmt.Errorf("get parent image: %w", err) - } - - return img, nil -} - func (s *OCIStore) Attach(ctx context.Context, parentDigest, artifactType string, payload []byte, checksum, targetPlatform, predicateType string) error { return s.AttachSuperseding(ctx, parentDigest, artifactType, payload, checksum, targetPlatform, predicateType, nil) } From aa8891e7694dda76237bf2ef874e544410ad40a5 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 17:23:41 +0300 Subject: [PATCH 5/6] fix(sbom): keep imageSaver out of the mockgen source file sbom_step.go carries a mockgen -source directive, so every interface declared in it gets a generated mock under test/mock, and mock:check in CI failed on the one imageSaver would have had. Its test double is a stub, not a mock; the interface moves to its own file. Signed-off-by: Radmir Khurum --- pkg/build/sbom_step.go | 4 ---- pkg/build/sbom_step_saved_image.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 pkg/build/sbom_step_saved_image.go diff --git a/pkg/build/sbom_step.go b/pkg/build/sbom_step.go index 4eedd40194..c73a706f0a 100644 --- a/pkg/build/sbom_step.go +++ b/pkg/build/sbom_step.go @@ -269,10 +269,6 @@ func recordShippedFiles(ctx context.Context, bom *cdx.BOM, backend imageSaver, w logboek.Context(ctx).Debug().LogF("Collected %d component filesets of the image, recorded onto %d components\n", len(filesets.Filesets), recorded) } -type imageSaver interface { - SaveImageToStream(ctx context.Context, imageName string) (io.ReadCloser, error) -} - // savedImage reads the image out of the backend as a docker-save tarball. The tarball is // read more than once, once for its manifest and once per layer, which a stream cannot // give, so it goes to a file first, the same way MutateAndPushImage does. The returned diff --git a/pkg/build/sbom_step_saved_image.go b/pkg/build/sbom_step_saved_image.go new file mode 100644 index 0000000000..0abd30df29 --- /dev/null +++ b/pkg/build/sbom_step_saved_image.go @@ -0,0 +1,12 @@ +package build + +import ( + "context" + "io" +) + +// Kept out of sbom_step.go: the mockgen directive there generates a mock for every +// interface in its source file, and a test double for this one is a stub, not a mock. +type imageSaver interface { + SaveImageToStream(ctx context.Context, imageName string) (io.ReadCloser, error) +} From 0e8888add9e1b66b2f1ae91afd5b3b68fc13d988 Mon Sep 17 00:00:00 2001 From: Radmir Khurum Date: Mon, 19 Oct 2026 17:33:02 +0300 Subject: [PATCH 6/6] test(sbom): prove a real build records the files of a pip package recordShippedFiles is fail-open, so a green e2e suite only showed the record breaks nothing; nothing showed it happens. This spec builds inject/pip_simple through werf, reads the SBOM back out of the registry and asserts requests carries occurrences: absolute paths of its own files, each with the file digest or the symlink marker, among them requests/__init__.py. With the call to recordShippedFiles removed the spec fails on "requests carries no evidence"; with it in place it passes on the Docker backend. Signed-off-by: Radmir Khurum --- test/e2e/sbom/pip_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/e2e/sbom/pip_test.go b/test/e2e/sbom/pip_test.go index 85513f7163..3acf317e2a 100644 --- a/test/e2e/sbom/pip_test.go +++ b/test/e2e/sbom/pip_test.go @@ -52,4 +52,37 @@ var _ = Describe("SBOM python-pip packages", Label("e2e", "sbom", "pip", "simple Expect(requests).NotTo(BeNil(), "expected requests@2.32.3 in BOM: the project pip.py shadowed the installer and the packages stage reported success without installing anything") }) + + It("records which files of the image each pip package consists of", func(ctx SpecContext) { + setupSbomBuildEnv() + + repoDirname := "repo_sbom_python_pip_shipped_files" + SuiteData.InitTestRepo(ctx, repoDirname, "inject/pip_simple") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{}}) + + sbomOut := werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{"app"}, + }, + }) + + bom := sbomtest.MustParseSBOMOutput(sbomOut) + requests := sbomtest.FindComponent(bom, "requests", "2.32.3") + Expect(requests).NotTo(BeNil(), "expected requests@2.32.3 in BOM") + Expect(requests.Evidence).NotTo(BeNil(), "requests carries no evidence: the files it consists of were not recorded") + Expect(requests.Evidence.Occurrences).NotTo(BeNil(), "requests carries no occurrences: the files it consists of were not recorded") + + occurrences := *requests.Evidence.Occurrences + Expect(occurrences).NotTo(BeEmpty(), "requests is installed, so its files are in the image") + for _, occurrence := range occurrences { + Expect(occurrence.Location).To(HavePrefix("/"), "an occurrence names an absolute path in the image") + Expect(occurrence.Location).To(ContainSubstring("requests"), "an occurrence of requests names one of its files") + Expect(occurrence.AdditionalContext).To(Or(MatchRegexp(`^sha256:[0-9a-f]{64}$`), Equal("symlink")), + "every occurrence werf writes carries the file digest or marks a symlink") + } + Expect(occurrences).To(ContainElement(HaveField("Location", HaveSuffix("/requests/__init__.py")))) + }) })