From 23ed256d7a3abc7d827929aceaa091340d62ba43 Mon Sep 17 00:00:00 2001 From: abhinavgautam01 Date: Sun, 18 Oct 2026 20:47:44 +0530 Subject: [PATCH 1/2] Add Chef Supermarket proxy support for Berkshelf and Policyfiles --- README.md | 21 +- cmd/proxy/main.go | 1 + cmd/proxy/main_test.go | 1 + config.example.yaml | 4 + docs/configuration.md | 12 + internal/config/config.go | 7 + internal/config/config_test.go | 4 + internal/handler/chef.go | 342 +++++++++++++++ internal/handler/chef_test.go | 424 +++++++++++++++++++ internal/handler/notfound_ecosystems_test.go | 2 + internal/server/dashboard.go | 13 + internal/server/middleware.go | 2 +- internal/server/middleware_test.go | 2 +- internal/server/server.go | 3 + 14 files changed, 835 insertions(+), 3 deletions(-) create mode 100644 internal/handler/chef.go create mode 100644 internal/handler/chef_test.go diff --git a/README.md b/README.md index 559f1a80..0fdb94b4 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | RPM | RHEL/Fedora | | ✓ | | Alpine | Alpine Linux | | ✓ | | Arch | Arch Linux | | ✓ | -| Chef | Chef | | ✗ | +| Chef Supermarket | Chef | | ✓ | | Generic | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | @@ -369,6 +369,24 @@ conan remote disable conancenter Or configure in `~/.conan2/remotes.json`. +### Chef Supermarket (Berkshelf, Policyfiles) + +Point Berkshelf at the proxy in your `Berksfile`: + +```ruby +source "http://localhost:8080/chef" +``` + +Or set the default source of a Policyfile: + +```ruby +default_source :supermarket, "http://localhost:8080/chef" +``` + +The proxy serves the `/universe` dependency index and the cookbook API, rewriting every download location to itself while leaving dependency constraints unchanged. Cookbook archives are fetched through the Supermarket download redirect and cached, so they stay available when the Supermarket is down. With `cache_metadata` on, the universe and cookbook metadata follow the usual TTL and stale fallback. + +To proxy a private Supermarket instead of `supermarket.chef.io`, set `upstream.chef` (or `PROXY_UPSTREAM_CHEF`) and add credentials under `upstream.auth` if it needs them. See [docs/configuration.md](docs/configuration.md#upstream-registries). + ### Conda Configure in `~/.condarc`: @@ -1114,6 +1132,7 @@ Recently cached: | `GET /nuget/*` | NuGet V3 API | | `GET /composer/*` | Composer/Packagist protocol | | `GET /conan/*` | Conan C/C++ protocol | +| `GET /chef/*` | Chef Supermarket protocol (universe, cookbook API and downloads) | | `GET /conda/*` | Conda/Anaconda protocol | | `GET /cran/*` | CRAN (R) protocol | | `GET /julia/*` | Julia Pkg server protocol | diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 75f00978..b42032d3 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -250,6 +250,7 @@ func runServe() { fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER Packagist API upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_COMPOSER_REPOSITORY Packagist repository upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONAN Conan registry upstream URL\n") + fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CHEF Chef Supermarket upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CONDA Conda channel upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_CRAN CRAN mirror upstream URL\n") fmt.Fprintf(os.Stderr, " PROXY_UPSTREAM_JULIA Julia package server upstream URL\n") diff --git a/cmd/proxy/main_test.go b/cmd/proxy/main_test.go index 4393c44b..3f17c278 100644 --- a/cmd/proxy/main_test.go +++ b/cmd/proxy/main_test.go @@ -35,6 +35,7 @@ func TestServeHelpListsUpstreamEnvironmentVariables(t *testing.T) { "PROXY_UPSTREAM_COMPOSER", "PROXY_UPSTREAM_COMPOSER_REPOSITORY", "PROXY_UPSTREAM_CONAN", + "PROXY_UPSTREAM_CHEF", "PROXY_UPSTREAM_CONDA", "PROXY_UPSTREAM_CRAN", "PROXY_UPSTREAM_JULIA", diff --git a/config.example.yaml b/config.example.yaml index accae89a..07f989ff 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -224,6 +224,10 @@ upstream: # Conan registry URL conan: "https://center.conan.io" + # Chef Supermarket URL (used by /chef/). Point it at a private + # Supermarket to proxy that instead. + chef: "https://supermarket.chef.io" + # Conda channel base URL conda: "https://conda.anaconda.org" diff --git a/docs/configuration.md b/docs/configuration.md index ffd57fe2..c53df79c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -250,6 +250,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under | `upstream.composer` | `PROXY_UPSTREAM_COMPOSER` | `https://packagist.org` | | `upstream.composer_repository` | `PROXY_UPSTREAM_COMPOSER_REPOSITORY` | `https://repo.packagist.org` | | `upstream.conan` | `PROXY_UPSTREAM_CONAN` | `https://center.conan.io` | +| `upstream.chef` | `PROXY_UPSTREAM_CHEF` | `https://supermarket.chef.io` | | `upstream.conda` | `PROXY_UPSTREAM_CONDA` | `https://conda.anaconda.org` | | `upstream.cran` | `PROXY_UPSTREAM_CRAN` | `https://cloud.r-project.org` | | `upstream.julia` | `PROXY_UPSTREAM_JULIA` | `https://pkg.julialang.org` | @@ -284,6 +285,17 @@ upstream: `upstream.hex_api` is used for cooldown timestamps and must expose Hex's `/api/packages/{name}` JSON endpoint. +`upstream.chef` is the Supermarket served at `/chef/`, for Berkshelf (`source "https://proxy.example.com/chef"`) and Policyfiles (`default_source :supermarket, "https://proxy.example.com/chef"`). The upstream has to serve the Supermarket API: `/universe`, `/api/v1/cookbooks/{name}`, its versions and their downloads. A private Supermarket that needs credentials takes them from `upstream.auth`, scoped to its URL. A download redirect to another host, such as the storage bucket behind Supermarket, does not receive them: + +```yaml +upstream: + chef: "https://supermarket.example.com" + auth: + "https://supermarket.example.com": + type: bearer + token: "${SUPERMARKET_TOKEN}" +``` + ### Private package routes Clients that use the proxy as their only registry can still install private packages. A package route sends every package whose name matches a pattern to a private registry, and never to the default upstream: diff --git a/internal/config/config.go b/internal/config/config.go index a9d7c212..b1f59317 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -653,6 +653,11 @@ type UpstreamConfig struct { // Default: https://center.conan.io Conan string `json:"conan" yaml:"conan"` + // Chef is the upstream Chef Supermarket URL, served at /chef/. Point it + // at a private Supermarket to proxy that instead of the public one. + // Default: https://supermarket.chef.io + Chef string `json:"chef" yaml:"chef"` + // Conda is the upstream Conda channel base URL. // Default: https://conda.anaconda.org Conda string `json:"conda" yaml:"conda"` @@ -1052,6 +1057,7 @@ func Default() *Config { Composer: "https://packagist.org", ComposerRepository: "https://repo.packagist.org", Conan: "https://center.conan.io", + Chef: "https://supermarket.chef.io", Conda: "https://conda.anaconda.org", CRAN: "https://cloud.r-project.org", Julia: "https://pkg.julialang.org", @@ -1195,6 +1201,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Upstream.Composer, "PROXY_UPSTREAM_COMPOSER") setEnvString(&c.Upstream.ComposerRepository, "PROXY_UPSTREAM_COMPOSER_REPOSITORY") setEnvString(&c.Upstream.Conan, "PROXY_UPSTREAM_CONAN") + setEnvString(&c.Upstream.Chef, "PROXY_UPSTREAM_CHEF") setEnvString(&c.Upstream.Conda, "PROXY_UPSTREAM_CONDA") setEnvString(&c.Upstream.CRAN, "PROXY_UPSTREAM_CRAN") setEnvString(&c.Upstream.Julia, "PROXY_UPSTREAM_JULIA") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e3b0a2c5..c941fcfe 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -33,6 +33,7 @@ func upstreamConfigValues(upstream UpstreamConfig) map[string]string { "composer": upstream.Composer, "composer_repository": upstream.ComposerRepository, "conan": upstream.Conan, + "chef": upstream.Chef, "conda": upstream.Conda, "cran": upstream.CRAN, "julia": upstream.Julia, @@ -64,6 +65,7 @@ func defaultUpstreamValues() map[string]string { "composer": "https://packagist.org", "composer_repository": "https://repo.packagist.org", "conan": "https://center.conan.io", + "chef": "https://supermarket.chef.io", "conda": "https://conda.anaconda.org", "cran": "https://cloud.r-project.org", "julia": "https://pkg.julialang.org", @@ -95,6 +97,7 @@ func upstreamEnvironmentVariables() map[string]string { "composer": "PROXY_UPSTREAM_COMPOSER", "composer_repository": "PROXY_UPSTREAM_COMPOSER_REPOSITORY", "conan": "PROXY_UPSTREAM_CONAN", + "chef": "PROXY_UPSTREAM_CHEF", "conda": "PROXY_UPSTREAM_CONDA", "cran": "PROXY_UPSTREAM_CRAN", "julia": "PROXY_UPSTREAM_JULIA", @@ -385,6 +388,7 @@ upstream: composer: "https://upstream.example.com/composer" composer_repository: "https://upstream.example.com/composer_repository" conan: "https://upstream.example.com/conan" + chef: "https://upstream.example.com/chef" conda: "https://upstream.example.com/conda" cran: "https://upstream.example.com/cran" julia: "https://upstream.example.com/julia" diff --git a/internal/handler/chef.go b/internal/handler/chef.go new file mode 100644 index 00000000..127d7973 --- /dev/null +++ b/internal/handler/chef.go @@ -0,0 +1,342 @@ +package handler + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "path" + "regexp" + "strings" +) + +const ( + chefEcosystem = "chef" + chefMountPath = "/chef" + chefAPIPath = "/api/v1" + // chefUpstreamKeyLength is how much of the upstream URL's digest prefixes + // metadata cache keys: enough to keep two upstreams apart. + chefUpstreamKeyLength = 16 +) + +var ( + // chefCookbookName matches Supermarket cookbook names: letters, digits, + // hyphens, underscores and dots, not starting with a dot. + chefCookbookName = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9_.-]*$`) + // chefVersion matches Chef cookbook versions, which have two or three + // numeric parts. + chefVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`) + + errChefDocument = errors.New("unexpected Chef Supermarket document") +) + +// ChefHandler serves a read-only Chef Supermarket at /chef. Berkshelf and +// Policyfiles resolve dependencies against /universe and then download each +// cookbook through the cookbook API. Every download location in those +// documents is rewritten to the proxy, so cookbooks are fetched through it and +// cached; dependency constraints and every other field pass through unchanged. +type ChefHandler struct { + proxy *Proxy + proxyURL string + upstreamURL string +} + +// NewChefHandler creates a Chef Supermarket handler for the Supermarket at +// upstreamURL, such as https://supermarket.chef.io or a private Supermarket. +func NewChefHandler(proxy *Proxy, proxyURL, upstreamURL string) *ChefHandler { + return &ChefHandler{ + proxy: proxy, + proxyURL: strings.TrimSuffix(proxyURL, "/"), + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + } +} + +// Routes returns the HTTP handler for Chef Supermarket requests. +func (h *ChefHandler) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /universe", h.handleUniverse) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}", h.handleCookbook) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}", h.handleVersion) + mux.HandleFunc("GET "+chefAPIPath+"/cookbooks/{name}/versions/{version}/download", h.handleDownload) + return mux +} + +func (h *ChefHandler) handleUniverse(w http.ResponseWriter, r *http.Request) { + h.serveDocument(w, r, "universe", h.upstreamURL+"/universe", h.rewriteUniverse) +} + +func (h *ChefHandler) handleCookbook(w http.ResponseWriter, r *http.Request) { + name, ok := chefCookbookFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook name", http.StatusBadRequest) + return + } + h.serveDocument(w, r, "cookbooks/"+name, h.upstreamCookbookURL(name), func(body []byte) ([]byte, error) { + return h.rewriteCookbook(name, body) + }) +} + +func (h *ChefHandler) handleVersion(w http.ResponseWriter, r *http.Request) { + name, version, ok := chefVersionFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook version", http.StatusBadRequest) + return + } + h.serveDocument(w, r, "cookbooks/"+name+"/versions/"+version, h.upstreamVersionURL(name, version), func(body []byte) ([]byte, error) { + return h.rewriteVersion(name, version, body) + }) +} + +// handleDownload serves a cookbook archive from the cache. An archive not +// cached yet is fetched through the Supermarket download endpoint, following +// the redirect it answers with. A cached archive is served without the +// upstream. +func (h *ChefHandler) handleDownload(w http.ResponseWriter, r *http.Request) { + name, version, ok := chefVersionFromRequest(r) + if !ok { + http.Error(w, "invalid cookbook version", http.StatusBadRequest) + return + } + + filename := name + "-" + version + ".tgz" + result, err := h.proxy.GetOrFetchArtifactFromURL( + r.Context(), chefEcosystem, name, version, filename, h.upstreamVersionURL(name, version)+"/download") + if err != nil { + h.proxy.serveArtifactError(w, err, "failed to fetch cookbook") + return + } + if result.Artifact.MediaType == "" { + w.Header().Set(headerContentType, "application/gzip") + } + ServeArtifactRequest(w, r, result) +} + +// serveDocument serves a JSON document from the metadata cache, which applies +// the metadata TTL and falls back to the stored copy while the upstream is +// unavailable, rewritten by rewrite. +func (h *ChefHandler) serveDocument(w http.ResponseWriter, r *http.Request, documentKey, upstreamURL string, rewrite func([]byte) ([]byte, error)) { + cacheKey := h.cacheKey(documentKey) + if stored, ok := h.proxy.storedRewrite(chefEcosystem, cacheKey, h.proxyURL, cacheKey); ok { + serveRewrittenMetadata(w, r, stored) + return + } + + upstream := h.proxy.fetchMetadataDocument(r.Context(), chefEcosystem, cacheKey, upstreamURL, contentTypeJSON) + if err := upstream.err; err != nil { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("Chef Supermarket request failed", "url", upstreamURL, "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + + rewritten, err := h.proxy.cachedRewrite(r.Context(), chefEcosystem, h.proxyURL, cacheKey, upstream.body, upstream.digest, + func(body []byte) ([]byte, []string, error) { + out, err := rewrite(body) + return out, nil, err + }) + if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } + // Serving the document unrewritten would send clients straight to + // the upstream, bypassing the cache. + h.proxy.Logger.Warn("failed to rewrite Chef Supermarket document", "url", upstreamURL, "error", err) + http.Error(w, "invalid Chef Supermarket response", http.StatusBadGateway) + return + } + serveRewrittenMetadata(w, r, rewritten) +} + +// rewriteUniverse points every version's location_path and download_url at +// the proxy. The universe maps cookbook names to versions to their location +// and dependencies; dependencies and any other field are copied as they are. +func (h *ChefHandler) rewriteUniverse(body []byte) ([]byte, error) { + var universe map[string]map[string]map[string]json.RawMessage + if err := json.Unmarshal(body, &universe); err != nil { + return nil, fmt.Errorf("%w: universe: %w", errChefDocument, err) + } + + locationPath, err := jsonString(h.apiURL()) + if err != nil { + return nil, err + } + for name, versions := range universe { + for version, entry := range versions { + if entry == nil { + continue + } + downloadURL, err := jsonString(h.downloadURL(name, version)) + if err != nil { + return nil, err + } + entry["location_path"] = locationPath + entry["download_url"] = downloadURL + } + } + return encodeChefJSON(universe) +} + +// rewriteCookbook points a cookbook's version URLs at the proxy. +func (h *ChefHandler) rewriteCookbook(name string, body []byte) ([]byte, error) { + var cookbook map[string]json.RawMessage + if err := json.Unmarshal(body, &cookbook); err != nil { + return nil, fmt.Errorf("%w: cookbook %s: %w", errChefDocument, name, err) + } + + if raw, ok := cookbook["latest_version"]; ok { + rewritten, err := h.rewriteVersionURL(name, raw) + if err != nil { + return nil, err + } + cookbook["latest_version"] = rewritten + } + if raw, ok := cookbook["versions"]; ok { + var versions []json.RawMessage + if err := json.Unmarshal(raw, &versions); err != nil { + return nil, fmt.Errorf("%w: cookbook %s versions: %w", errChefDocument, name, err) + } + for i, versionURL := range versions { + rewritten, err := h.rewriteVersionURL(name, versionURL) + if err != nil { + return nil, err + } + versions[i] = rewritten + } + encoded, err := encodeChefJSON(versions) + if err != nil { + return nil, err + } + cookbook["versions"] = encoded + } + return encodeChefJSON(cookbook) +} + +// rewriteVersion points a cookbook version's cookbook and file URLs at the +// proxy. +func (h *ChefHandler) rewriteVersion(name, version string, body []byte) ([]byte, error) { + var document map[string]json.RawMessage + if err := json.Unmarshal(body, &document); err != nil { + return nil, fmt.Errorf("%w: cookbook %s version %s: %w", errChefDocument, name, version, err) + } + + if _, ok := document["cookbook"]; ok { + cookbookURL, err := jsonString(h.cookbookURL(name)) + if err != nil { + return nil, err + } + document["cookbook"] = cookbookURL + } + if _, ok := document["file"]; ok { + fileURL, err := jsonString(h.downloadURL(name, version)) + if err != nil { + return nil, err + } + document["file"] = fileURL + } + return encodeChefJSON(document) +} + +// rewriteVersionURL rewrites an upstream cookbook version URL, a JSON string +// ending in /versions/{version}, to the proxy. Anything else, such as null, +// is returned unchanged. +func (h *ChefHandler) rewriteVersionURL(name string, raw json.RawMessage) (json.RawMessage, error) { + var versionURL string + if err := json.Unmarshal(raw, &versionURL); err != nil { + return raw, nil //nolint:nilerr // not a string, so not a URL to rewrite + } + parsed, err := url.Parse(versionURL) + if err != nil { + return raw, nil //nolint:nilerr // not a URL, so nothing to rewrite + } + version, ok := normalizeChefVersion(path.Base(parsed.Path)) + if !ok { + return raw, nil + } + return jsonString(h.versionURL(name, version)) +} + +func (h *ChefHandler) apiURL() string { + return h.proxyURL + chefMountPath + chefAPIPath +} + +func (h *ChefHandler) cookbookURL(name string) string { + return h.apiURL() + "/cookbooks/" + url.PathEscape(name) +} + +func (h *ChefHandler) versionURL(name, version string) string { + return h.cookbookURL(name) + "/versions/" + url.PathEscape(version) +} + +func (h *ChefHandler) downloadURL(name, version string) string { + return h.versionURL(name, version) + "/download" +} + +func (h *ChefHandler) upstreamCookbookURL(name string) string { + return h.upstreamURL + chefAPIPath + "/cookbooks/" + url.PathEscape(name) +} + +func (h *ChefHandler) upstreamVersionURL(name, version string) string { + return h.upstreamCookbookURL(name) + "/versions/" + url.PathEscape(version) +} + +// cacheKey prefixes a metadata cache key with the upstream's digest, so +// pointing upstream.chef at another Supermarket does not serve the previous +// one's documents. +func (h *ChefHandler) cacheKey(documentKey string) string { + digest := sha256.Sum256([]byte(h.upstreamURL)) + return hex.EncodeToString(digest[:])[:chefUpstreamKeyLength] + "/" + documentKey +} + +func chefCookbookFromRequest(r *http.Request) (string, bool) { + name := r.PathValue("name") + if !chefCookbookName.MatchString(name) || containsPathTraversal(name) { + return "", false + } + return name, true +} + +func chefVersionFromRequest(r *http.Request) (name, version string, ok bool) { + name, ok = chefCookbookFromRequest(r) + if !ok { + return "", "", false + } + version, ok = normalizeChefVersion(r.PathValue("version")) + return name, version, ok +} + +// normalizeChefVersion accepts a version in either form Supermarket serves +// (1.2.3, or 1_2_3 as Berkshelf writes it in URLs) and returns the dotted +// form. +func normalizeChefVersion(version string) (string, bool) { + version = strings.ReplaceAll(version, "_", ".") + if !chefVersion.MatchString(version) { + return "", false + } + return version, true +} + +func jsonString(s string) (json.RawMessage, error) { + return encodeChefJSON(s) +} + +// encodeChefJSON encodes v without escaping HTML characters, so dependency +// constraints such as ">= 1.0" keep their upstream spelling. +func encodeChefJSON(v any) (json.RawMessage, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return nil, err + } + return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil +} diff --git a/internal/handler/chef_test.go b/internal/handler/chef_test.go new file mode 100644 index 00000000..11aedbf1 --- /dev/null +++ b/internal/handler/chef_test.go @@ -0,0 +1,424 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" + "github.com/git-pkgs/registries/fetch" +) + +const ( + chefTestProxyURL = "http://proxy.example" + chefTestAPI = chefTestProxyURL + "/chef/api/v1" + chefTestTarball = "cookbook tarball bytes" +) + +// fakeSupermarket serves the parts of the Supermarket API Berkshelf uses, in +// the shapes supermarket.chef.io returns them. Downloads redirect to a second +// server standing in for the storage host, as Supermarket redirects to S3. +type fakeSupermarket struct { + api *httptest.Server + files *httptest.Server + // filesURL is files.URL on another hostname, so a redirect to it leaves + // the API host the way a redirect to S3 does. + filesURL string + + available atomic.Bool + // token, when set, is the bearer token the API requires. + token string + + mu sync.Mutex + requests map[string]int + fileAuth []string +} + +func newFakeSupermarket(t *testing.T, token string) *fakeSupermarket { + t.Helper() + s := &fakeSupermarket{token: token, requests: map[string]int{}} + s.available.Store(true) + + s.files = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests["files "+r.URL.Path]++ + s.fileAuth = append(s.fileAuth, r.Header.Get("Authorization")) + s.mu.Unlock() + if !s.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if r.URL.Path != "/cookbook_versions/tarballs/34899/original/apt.tgz" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-gzip") + _, _ = w.Write([]byte(chefTestTarball)) + })) + t.Cleanup(s.files.Close) + s.filesURL = strings.Replace(s.files.URL, "127.0.0.1", "localhost", 1) + + s.api = httptest.NewServer(http.HandlerFunc(s.serveAPI)) + t.Cleanup(s.api.Close) + return s +} + +func (s *fakeSupermarket) serveAPI(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests[r.URL.Path]++ + s.mu.Unlock() + if !s.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + if s.token != "" && r.Header.Get("Authorization") != "Bearer "+s.token { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + api := s.api.URL + "/api/v1" + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/universe": + // Like Supermarket's, these locations carry an explicit port. + located := api + _, _ = fmt.Fprintf(w, `{"apt":{"7.5.22":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"compat_resource":">= 0.0.0"}},`+ + `"7.6.0":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{}}},`+ + `"compat_resource":{"12.19.1":{"location_type":"opscode","location_path":%q,"download_url":%q,"dependencies":{"apt":"~> 7.5"}}}}`, + located, api+"/cookbooks/apt/versions/7.5.22/download", + located, api+"/cookbooks/apt/versions/7.6.0/download", + located, api+"/cookbooks/compat_resource/versions/12.19.1/download") + case "/api/v1/cookbooks/apt": + _, _ = fmt.Fprintf(w, `{"name":"apt","maintainer":"sous-chefs","latest_version":%q,"external_url":"https://github.com/sous-chefs/apt","deprecated":false,"versions":[%q,%q]}`, + api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.5.22") + case "/api/v1/cookbooks/apt/versions/7.5.22": + _, _ = fmt.Fprintf(w, `{"license":"Apache-2.0","tarball_file_size":%d,"version":"7.5.22","cookbook":%q,"file":%q,"dependencies":{"compat_resource":">= 0.0.0"}}`, + len(chefTestTarball), api+"/cookbooks/apt", api+"/cookbooks/apt/versions/7.5.22/download") + case "/api/v1/cookbooks/apt/versions/7.5.22/download": + http.Redirect(w, r, s.filesURL+"/cookbook_versions/tarballs/34899/original/apt.tgz?1714639378", http.StatusFound) + default: + http.NotFound(w, r) + } +} + +func (s *fakeSupermarket) count(key string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requests[key] +} + +// newChefTestHandler returns a handler for s using client for every upstream +// request, with metadata caching on. +func newChefTestHandler(t *testing.T, s *fakeSupermarket, client *http.Client) (*ChefHandler, *Proxy) { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = client + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(client), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + return NewChefHandler(proxy, chefTestProxyURL, s.api.URL), proxy +} + +func serveChefRequest(h *ChefHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} + +// proxyPath turns an absolute proxy URL from a rewritten document into the +// path the handler is mounted at. +func proxyPath(t *testing.T, rawURL string) string { + t.Helper() + parsed, err := url.Parse(rawURL) + if err != nil { + t.Fatalf("parse %q: %v", rawURL, err) + } + if parsed.Scheme+"://"+parsed.Host != chefTestProxyURL || !strings.HasPrefix(parsed.Path, "/chef/") { + t.Fatalf("URL %q does not point at the proxy", rawURL) + } + return strings.TrimPrefix(parsed.Path, "/chef") +} + +type chefUniverseEntry struct { + LocationType string `json:"location_type"` + LocationPath string `json:"location_path"` + DownloadURL string `json:"download_url"` + Dependencies json.RawMessage `json:"dependencies"` +} + +func decodeChefUniverse(t *testing.T, body []byte) map[string]map[string]chefUniverseEntry { + t.Helper() + var universe map[string]map[string]chefUniverseEntry + if err := json.Unmarshal(body, &universe); err != nil { + t.Fatalf("decode universe: %v\n%s", err, body) + } + return universe +} + +func TestChefUniverseRewritesLocations(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + w := serveChefRequest(h, "/universe") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if got := w.Header().Get("Content-Type"); got != contentTypeJSON { + t.Errorf("Content-Type = %q, want %q", got, contentTypeJSON) + } + body := w.Body.String() + if strings.Contains(body, s.api.URL) { + t.Errorf("rewritten universe still points at the upstream: %s", body) + } + + universe := decodeChefUniverse(t, w.Body.Bytes()) + wantDeps := map[string]map[string]string{ + "apt": {"7.5.22": `{"compat_resource":">= 0.0.0"}`, "7.6.0": `{}`}, + "compat_resource": {"12.19.1": `{"apt":"~> 7.5"}`}, + } + for name, versions := range wantDeps { + for version, deps := range versions { + entry, ok := universe[name][version] + if !ok { + t.Errorf("universe lost %s %s", name, version) + continue + } + if entry.LocationType != "opscode" { + t.Errorf("%s %s location_type = %q, want opscode", name, version, entry.LocationType) + } + if entry.LocationPath != chefTestAPI { + t.Errorf("%s %s location_path = %q, want %q", name, version, entry.LocationPath, chefTestAPI) + } + wantDownload := chefTestAPI + "/cookbooks/" + name + "/versions/" + version + "/download" + if entry.DownloadURL != wantDownload { + t.Errorf("%s %s download_url = %q, want %q", name, version, entry.DownloadURL, wantDownload) + } + if string(entry.Dependencies) != deps { + t.Errorf("%s %s dependencies = %s, want %s unchanged", name, version, entry.Dependencies, deps) + } + } + } + // Constraints keep their spelling rather than being HTML-escaped. + if !strings.Contains(body, `">= 0.0.0"`) || !strings.Contains(body, `"~> 7.5"`) { + t.Errorf("dependency constraints were re-escaped: %s", body) + } +} + +func TestChefCookbookMetadataRewritesVersionURLs(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + w := serveChefRequest(h, "/api/v1/cookbooks/apt") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var cookbook struct { + Name string `json:"name"` + LatestVersion string `json:"latest_version"` + Versions []string `json:"versions"` + ExternalURL string `json:"external_url"` + Deprecated bool `json:"deprecated"` + } + if err := json.Unmarshal(w.Body.Bytes(), &cookbook); err != nil { + t.Fatalf("decode: %v", err) + } + if cookbook.LatestVersion != chefTestAPI+"/cookbooks/apt/versions/7.6.0" { + t.Errorf("latest_version = %q", cookbook.LatestVersion) + } + want := []string{chefTestAPI + "/cookbooks/apt/versions/7.6.0", chefTestAPI + "/cookbooks/apt/versions/7.5.22"} + if strings.Join(cookbook.Versions, ",") != strings.Join(want, ",") { + t.Errorf("versions = %v, want %v", cookbook.Versions, want) + } + if cookbook.Name != "apt" || cookbook.ExternalURL != "https://github.com/sous-chefs/apt" || cookbook.Deprecated { + t.Errorf("unrelated fields changed: %+v", cookbook) + } +} + +func TestChefVersionMetadataAcceptsBerkshelfVersions(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + // Berkshelf writes versions with underscores in cookbook API paths. + for _, version := range []string{"7_5_22", "7.5.22"} { + w := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/"+version) + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", version, w.Code, w.Body.String()) + } + var document struct { + Version string `json:"version"` + Cookbook string `json:"cookbook"` + File string `json:"file"` + Dependencies json.RawMessage `json:"dependencies"` + License string `json:"license"` + } + if err := json.Unmarshal(w.Body.Bytes(), &document); err != nil { + t.Fatalf("%s: decode: %v", version, err) + } + if document.Cookbook != chefTestAPI+"/cookbooks/apt" { + t.Errorf("%s: cookbook = %q", version, document.Cookbook) + } + if document.File != chefTestAPI+"/cookbooks/apt/versions/7.5.22/download" { + t.Errorf("%s: file = %q", version, document.File) + } + if string(document.Dependencies) != `{"compat_resource":">= 0.0.0"}` || document.License != "Apache-2.0" || document.Version != "7.5.22" { + t.Errorf("%s: unrelated fields changed: %+v", version, document) + } + } + // Both spellings are one document, fetched once. + if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22"); got != 1 { + t.Errorf("upstream version requests = %d, want 1", got) + } +} + +// TestChefBerkshelfFlow follows the requests Berkshelf makes: the universe, +// then the cookbook version at the universe's location_path, then its file. +// The download redirects to the storage host and is cached. It is then served +// with the upstream down, as is the universe. +func TestChefBerkshelfFlow(t *testing.T) { + s := newFakeSupermarket(t, "") + h, proxy := newChefTestHandler(t, s, s.api.Client()) + + universe := decodeChefUniverse(t, serveChefRequest(h, "/universe").Body.Bytes()) + locationPath := universe["apt"]["7.5.22"].LocationPath + + versionResponse := serveChefRequest(h, proxyPath(t, locationPath+"/cookbooks/apt/versions/7_5_22")) + if versionResponse.Code != http.StatusOK { + t.Fatalf("version status = %d: %s", versionResponse.Code, versionResponse.Body.String()) + } + var version struct { + File string `json:"file"` + } + if err := json.Unmarshal(versionResponse.Body.Bytes(), &version); err != nil { + t.Fatalf("decode version: %v", err) + } + + download := serveChefRequest(h, proxyPath(t, version.File)) + if download.Code != http.StatusOK { + t.Fatalf("download status = %d: %s", download.Code, download.Body.String()) + } + if download.Body.String() != chefTestTarball { + t.Errorf("download body = %q, want the tarball", download.Body.String()) + } + if got := s.count("files /cookbook_versions/tarballs/34899/original/apt.tgz"); got != 1 { + t.Errorf("storage host requests = %d, want 1 after following the redirect", got) + } + + // A second download is a cache hit. + if again := serveChefRequest(h, proxyPath(t, version.File)); again.Code != http.StatusOK || again.Body.String() != chefTestTarball { + t.Fatalf("cached download = %d %q", again.Code, again.Body.String()) + } + if got := s.count("/api/v1/cookbooks/apt/versions/7.5.22/download"); got != 1 { + t.Errorf("upstream download requests = %d, want 1 (second served from cache)", got) + } + + // With the upstream down and the metadata TTL lapsed, the cached + // cookbook and the stored universe are still served. + s.available.Store(false) + proxy.MetadataTTL = 0 + if offline := serveChefRequest(h, proxyPath(t, version.File)); offline.Code != http.StatusOK || offline.Body.String() != chefTestTarball { + t.Errorf("offline download = %d %q, want the cached tarball", offline.Code, offline.Body.String()) + } + offlineUniverse := serveChefRequest(h, "/universe") + if offlineUniverse.Code != http.StatusOK { + t.Fatalf("offline universe status = %d: %s", offlineUniverse.Code, offlineUniverse.Body.String()) + } + if got := decodeChefUniverse(t, offlineUniverse.Body.Bytes())["apt"]["7.5.22"].DownloadURL; got != version.File { + t.Errorf("offline universe download_url = %q, want %q", got, version.File) + } +} + +func TestChefUpstreamAuthentication(t *testing.T) { + const token = "supermarket-token" + s := newFakeSupermarket(t, token) + // Credentials are scoped to the Supermarket, as upstream.auth scopes + // them by URL, so the storage host it redirects to never sees them. + client := &http.Client{Transport: upstreamhttp.NewTransport(http.DefaultTransport, + upstreamhttp.AuthFunc(func(requestURL string) (string, string) { + if strings.HasPrefix(requestURL, s.api.URL+"/") { + return "Authorization", "Bearer " + token + } + return "", "" + }))} + h, _ := newChefTestHandler(t, s, client) + + for _, target := range []string{"/universe", "/api/v1/cookbooks/apt", "/api/v1/cookbooks/apt/versions/7.5.22"} { + if w := serveChefRequest(h, target); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200: %s", target, w.Code, w.Body.String()) + } + } + download := serveChefRequest(h, "/api/v1/cookbooks/apt/versions/7.5.22/download") + if download.Code != http.StatusOK || download.Body.String() != chefTestTarball { + t.Fatalf("download = %d %q", download.Code, download.Body.String()) + } + + s.mu.Lock() + defer s.mu.Unlock() + for _, auth := range s.fileAuth { + if auth != "" { + t.Errorf("storage host received Authorization %q", auth) + } + } +} + +func TestChefRejectsInvalidRequests(t *testing.T) { + s := newFakeSupermarket(t, "") + h, _ := newChefTestHandler(t, s, s.api.Client()) + + tests := []struct { + target string + want int + }{ + {"/api/v1/cookbooks/.hidden", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/latest", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/1.2.3.4", http.StatusBadRequest}, + {"/api/v1/cookbooks/apt/versions/1.x/download", http.StatusBadRequest}, + {"/api/v1/cookbooks/missing", http.StatusNotFound}, + {"/api/v1/cookbooks/apt/versions/9.9.9", http.StatusNotFound}, + } + for _, tt := range tests { + if w := serveChefRequest(h, tt.target); w.Code != tt.want { + t.Errorf("%s: status = %d, want %d: %s", tt.target, w.Code, tt.want, w.Body.String()) + } + } +} + +// An upstream document that cannot be rewritten is refused rather than +// passed through, which would send clients around the proxy. +func TestChefUnreadableUniverseIsBadGateway(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`["not","a","universe"]`)) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + h := NewChefHandler(proxy, chefTestProxyURL, upstream.URL) + + w := serveChefRequest(h, "/universe") + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502: %s", w.Code, w.Body.String()) + } + if strings.Contains(w.Body.String(), "not") { + t.Errorf("unreadable document was passed through: %s", w.Body.String()) + } +} + +func TestChefCacheKeyDependsOnUpstream(t *testing.T) { + a := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.chef.io") + b := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com/") + if a.cacheKey("universe") == b.cacheKey("universe") { + t.Error("two Supermarkets share a universe cache key") + } + if containsPathTraversal(a.cacheKey("cookbooks/apt/versions/7.5.22")) { + t.Error("cache key is not a valid metadata key") + } +} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index e2141246..e8b3b9b0 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -39,6 +39,8 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { func(p *Proxy) http.Handler { return NewGemHandler(p, "http://localhost").Routes() }}, {"hex", "/tarballs/phoenix-1.7.10.tar", func(p *Proxy) http.Handler { return NewHexHandler(p, "http://localhost").Routes() }}, + {"chef", "/api/v1/cookbooks/apt/versions/7.5.22/download", + func(p *Proxy) http.Handler { return NewChefHandler(p, "http://localhost", "http://localhost").Routes() }}, } for _, tt := range tests { diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index dbc3a69d..597e6919 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -136,6 +136,7 @@ func supportedEcosystems() []string { "alpine", "arch", "cargo", + "chef", "composer", "conan", "conda", @@ -189,6 +190,8 @@ func ecosystemColorClasses(ecosystem string) string { return "bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" case "conan": return "bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" + case "chef": + return "bg-rose-100 text-rose-700 dark:bg-rose-900/50 dark:text-rose-300" case "conda": return "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" case "cran": @@ -411,6 +414,16 @@ composer config -g repositories.proxy composer ` + baseURL + `/composerConfigure Conan to use the proxy:

conan remote add proxy ` + baseURL + `/conan
 conan remote disable conancenter
`), + }, + { + ID: "chef", + Name: "Chef Supermarket", + Language: "Chef", + Endpoint: "/chef/", + Instructions: template.HTML(`

Point Berkshelf at the proxy in your Berksfile:

+
source "` + baseURL + `/chef"
+

Or in a Policyfile:

+
default_source :supermarket, "` + baseURL + `/chef"
`), }, { ID: "conda", diff --git a/internal/server/middleware.go b/internal/server/middleware.go index f74590a9..0de0d1bc 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -106,7 +106,7 @@ func requestEcosystem(path string) string { segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", - "conan", "conda", "cran", "julia", "debian", "rpm", "arch", + "conan", "chef", "conda", "cran", "julia", "debian", "rpm", "arch", "helm", "homebrew", "generic", "swift": return segment case "apk": diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 875b152e..7c9eab92 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -311,7 +311,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/npm/x": "npm", "/cargo/x": "cargo", "/gem/x": "rubygems", "/go/x": "golang", "/hex/x": "hex", "/pub/x": "pub", "/pypi/x": "pypi", "/maven/x": "maven", "/gradle/x": "gradle", "/nuget/x": "nuget", "/composer/x": "packagist", - "/conan/x": "conan", "/conda/x": "conda", "/cran/x": "cran", + "/conan/x": "conan", "/chef/x": "chef", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", diff --git a/internal/server/server.go b/internal/server/server.go index d9699cd4..2aab0f8f 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -13,6 +13,7 @@ // - /nuget/* - NuGet V3 API protocol // - /composer/* - Composer/Packagist protocol // - /conan/* - Conan C/C++ protocol +// - /chef/* - Chef Supermarket protocol // - /conda/* - Conda/Anaconda protocol // - /cran/* - CRAN (R) protocol // - /julia/* - Julia Pkg server protocol @@ -437,6 +438,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.ComposerRepository, ).WithPackageRoutes(s.cfg.Upstream.ComposerRoutes) conanHandler := handler.NewConanHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conan) + chefHandler := handler.NewChefHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Chef) condaHandler := handler.NewCondaHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.Conda) cranHandler := handler.NewCRANHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.CRAN) juliaHandler := handler.NewJuliaHandlerWithUpstream(proxy, s.cfg.Upstream.Julia) @@ -469,6 +471,7 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/nuget", http.StripPrefix("/nuget", nugetHandler.Routes())) r.Mount("/composer", http.StripPrefix("/composer", composerHandler.Routes())) r.Mount("/conan", http.StripPrefix("/conan", conanHandler.Routes())) + r.Mount("/chef", http.StripPrefix("/chef", chefHandler.Routes())) r.Mount("/conda", http.StripPrefix("/conda", condaHandler.Routes())) r.Mount("/cran", http.StripPrefix("/cran", cranHandler.Routes())) r.Mount("/julia", http.StripPrefix("/julia", juliaHandler.Routes())) From 558e9ea6953b4173ff5bf68a58172054c6e3be03 Mon Sep 17 00:00:00 2001 From: abhinavgautam01 Date: Mon, 19 Oct 2026 11:05:11 +0530 Subject: [PATCH 2/2] Scope cached Chef cookbook archives to the upstream Supermarket --- internal/handler/chef.go | 34 ++++++++++++++------- internal/handler/chef_test.go | 57 +++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/internal/handler/chef.go b/internal/handler/chef.go index 127d7973..72839920 100644 --- a/internal/handler/chef.go +++ b/internal/handler/chef.go @@ -18,8 +18,9 @@ const ( chefEcosystem = "chef" chefMountPath = "/chef" chefAPIPath = "/api/v1" - // chefUpstreamKeyLength is how much of the upstream URL's digest prefixes - // metadata cache keys: enough to keep two upstreams apart. + // chefUpstreamKeyLength is how much of the upstream URL's digest scopes + // metadata cache keys and archive filenames: enough to keep two upstreams + // apart. chefUpstreamKeyLength = 16 ) @@ -43,15 +44,22 @@ type ChefHandler struct { proxy *Proxy proxyURL string upstreamURL string + // upstreamKey identifies the upstream in cache keys, so pointing + // upstream.chef at another Supermarket does not serve the previous one's + // documents or archives. + upstreamKey string } // NewChefHandler creates a Chef Supermarket handler for the Supermarket at // upstreamURL, such as https://supermarket.chef.io or a private Supermarket. func NewChefHandler(proxy *Proxy, proxyURL, upstreamURL string) *ChefHandler { + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + digest := sha256.Sum256([]byte(upstreamURL)) return &ChefHandler{ proxy: proxy, proxyURL: strings.TrimSuffix(proxyURL, "/"), - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + upstreamURL: upstreamURL, + upstreamKey: hex.EncodeToString(digest[:])[:chefUpstreamKeyLength], } } @@ -94,7 +102,8 @@ func (h *ChefHandler) handleVersion(w http.ResponseWriter, r *http.Request) { // handleDownload serves a cookbook archive from the cache. An archive not // cached yet is fetched through the Supermarket download endpoint, following // the redirect it answers with. A cached archive is served without the -// upstream. +// upstream. Archives are cached per upstream, since two Supermarkets can +// publish different cookbooks under the same name and version. func (h *ChefHandler) handleDownload(w http.ResponseWriter, r *http.Request) { name, version, ok := chefVersionFromRequest(r) if !ok { @@ -102,9 +111,8 @@ func (h *ChefHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - filename := name + "-" + version + ".tgz" result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), chefEcosystem, name, version, filename, h.upstreamVersionURL(name, version)+"/download") + r.Context(), chefEcosystem, name, version, h.archiveFilename(name, version), h.upstreamVersionURL(name, version)+"/download") if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch cookbook") return @@ -289,12 +297,16 @@ func (h *ChefHandler) upstreamVersionURL(name, version string) string { return h.upstreamCookbookURL(name) + "/versions/" + url.PathEscape(version) } -// cacheKey prefixes a metadata cache key with the upstream's digest, so -// pointing upstream.chef at another Supermarket does not serve the previous -// one's documents. +// cacheKey prefixes a metadata cache key with the upstream's key. func (h *ChefHandler) cacheKey(documentKey string) string { - digest := sha256.Sum256([]byte(h.upstreamURL)) - return hex.EncodeToString(digest[:])[:chefUpstreamKeyLength] + "/" + documentKey + return h.upstreamKey + "/" + documentKey +} + +// archiveFilename names a cookbook archive in the cache. The filename is part +// of the artifact's cache identity, so it carries the upstream's key; the +// .tgz extension is kept for browsing the archive. +func (h *ChefHandler) archiveFilename(name, version string) string { + return name + "-" + version + "-" + h.upstreamKey + ".tgz" } func chefCookbookFromRequest(r *http.Request) (string, bool) { diff --git a/internal/handler/chef_test.go b/internal/handler/chef_test.go index 11aedbf1..51290a59 100644 --- a/internal/handler/chef_test.go +++ b/internal/handler/chef_test.go @@ -35,6 +35,8 @@ type fakeSupermarket struct { available atomic.Bool // token, when set, is the bearer token the API requires. token string + // tarball is the content of every cookbook archive it serves. + tarball string mu sync.Mutex requests map[string]int @@ -43,7 +45,7 @@ type fakeSupermarket struct { func newFakeSupermarket(t *testing.T, token string) *fakeSupermarket { t.Helper() - s := &fakeSupermarket{token: token, requests: map[string]int{}} + s := &fakeSupermarket{token: token, tarball: chefTestTarball, requests: map[string]int{}} s.available.Store(true) s.files = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -60,7 +62,7 @@ func newFakeSupermarket(t *testing.T, token string) *fakeSupermarket { return } w.Header().Set("Content-Type", "application/x-gzip") - _, _ = w.Write([]byte(chefTestTarball)) + _, _ = w.Write([]byte(s.tarball)) })) t.Cleanup(s.files.Close) s.filesURL = strings.Replace(s.files.URL, "127.0.0.1", "localhost", 1) @@ -100,7 +102,7 @@ func (s *fakeSupermarket) serveAPI(w http.ResponseWriter, r *http.Request) { api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.6.0", api+"/cookbooks/apt/versions/7.5.22") case "/api/v1/cookbooks/apt/versions/7.5.22": _, _ = fmt.Fprintf(w, `{"license":"Apache-2.0","tarball_file_size":%d,"version":"7.5.22","cookbook":%q,"file":%q,"dependencies":{"compat_resource":">= 0.0.0"}}`, - len(chefTestTarball), api+"/cookbooks/apt", api+"/cookbooks/apt/versions/7.5.22/download") + len(s.tarball), api+"/cookbooks/apt", api+"/cookbooks/apt/versions/7.5.22/download") case "/api/v1/cookbooks/apt/versions/7.5.22/download": http.Redirect(w, r, s.filesURL+"/cookbook_versions/tarballs/34899/original/apt.tgz?1714639378", http.StatusFound) default: @@ -412,6 +414,46 @@ func TestChefUnreadableUniverseIsBadGateway(t *testing.T) { } } +// Switching upstream.chef to another Supermarket keeps the cache, but an +// archive cached from the previous one is not served for a cookbook the new +// one publishes under the same name and version. +func TestChefArchiveCacheDependsOnUpstream(t *testing.T) { + const target = "/api/v1/cookbooks/apt/versions/7.5.22/download" + public := newFakeSupermarket(t, "") + private := newFakeSupermarket(t, "") + private.tarball = "private cookbook tarball bytes" + + publicHandler, proxy := newChefTestHandler(t, public, public.api.Client()) + if w := serveChefRequest(publicHandler, target); w.Code != http.StatusOK || w.Body.String() != public.tarball { + t.Fatalf("public download = %d %q", w.Code, w.Body.String()) + } + + privateHandler := NewChefHandler(proxy, chefTestProxyURL, private.api.URL) + w := serveChefRequest(privateHandler, target) + if w.Code != http.StatusOK { + t.Fatalf("private download status = %d: %s", w.Code, w.Body.String()) + } + if w.Body.String() != private.tarball { + t.Errorf("private download = %q, want the private Supermarket's tarball", w.Body.String()) + } + if got := private.count(target); got != 1 { + t.Errorf("private download requests = %d, want 1", got) + } + + // Each upstream's archive stays cached under its own identity. + for _, tt := range []struct { + handler *ChefHandler + upstream *fakeSupermarket + }{{publicHandler, public}, {privateHandler, private}} { + if again := serveChefRequest(tt.handler, target); again.Code != http.StatusOK || again.Body.String() != tt.upstream.tarball { + t.Errorf("cached download from %s = %d %q, want %q", tt.upstream.api.URL, again.Code, again.Body.String(), tt.upstream.tarball) + } + if got := tt.upstream.count(target); got != 1 { + t.Errorf("download requests to %s = %d, want 1 (second served from cache)", tt.upstream.api.URL, got) + } + } +} + func TestChefCacheKeyDependsOnUpstream(t *testing.T) { a := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.chef.io") b := NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com/") @@ -421,4 +463,13 @@ func TestChefCacheKeyDependsOnUpstream(t *testing.T) { if containsPathTraversal(a.cacheKey("cookbooks/apt/versions/7.5.22")) { t.Error("cache key is not a valid metadata key") } + if a.archiveFilename("apt", "7.5.22") == b.archiveFilename("apt", "7.5.22") { + t.Error("two Supermarkets share an archive filename") + } + if got := a.archiveFilename("apt", "7.5.22"); !strings.HasPrefix(got, "apt-7.5.22-") || !strings.HasSuffix(got, ".tgz") { + t.Errorf("archive filename = %q, want apt-7.5.22-.tgz", got) + } + if NewChefHandler(nil, chefTestProxyURL, "https://supermarket.example.com").upstreamKey != b.upstreamKey { + t.Error("a trailing slash changes the upstream key") + } }