diff --git a/README.md b/README.md index 3421dffb..aac9ff57 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | -| Maven | Java | | ✓ | +| Maven | Java | Yes‡ | ✓ | | Gradle Build Cache | Java/Kotlin | | ✓ | | NuGet | .NET | Yes | ✓ | | Composer | PHP | Yes | ✓ | @@ -79,6 +79,8 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. +‡ Maven cooldown uses the `Last-Modified` header of each version's `.pom`. On Maven Central that is the deploy time; a Nexus or Artifactory upstream may report when it cached the file instead. SNAPSHOT versions are not held back. + ## Install ```bash diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec69..226d23e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -980,10 +980,19 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. + +Maven caveats: + +- A Nexus or Artifactory upstream may report when it cached the `.pom` rather than when it was published, so a version can stay in cooldown longer than it should. +- On Google's Maven Central mirror, `Last-Modified` is when the file reached the mirror's bucket, a few hours after it was published, so versions stay in cooldown a little longer. The upload time is in `x-goog-meta-last-modified`, which the proxy does not read. It is missing for files from the mirror's 2019 bulk import. +- Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. +- SNAPSHOT versions are not held back. + With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8bcac24f..b5c34ef6 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1965,7 +1965,7 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco if p.StreamArtifacts { return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, func(fetchCtx context.Context) (*fetch.Artifact, error) { - return p.Fetcher.FetchWithHeaders(fetchCtx, downloadURL, headers) + return p.fetchUpstream(fetchCtx, downloadURL, headers) }) } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { @@ -2040,7 +2040,7 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) fetchStart := time.Now() - artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + artifact, err := p.fetchUpstream(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ @@ -2054,6 +2054,43 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) } +type upstreamLastModifiedKey struct{} + +// WithUpstreamLastModified returns a context under which an artifact fetched +// from upstream reports its Last-Modified to fn, so a handler can learn a +// version's publish time from the download itself rather than with a +// request of its own. fn is not called for a cache hit, or when the +// response has no usable Last-Modified. +func WithUpstreamLastModified(ctx context.Context, fn func(time.Time)) context.Context { + return context.WithValue(ctx, upstreamLastModifiedKey{}, fn) +} + +// observedFetcher is the part of fetch.Fetcher and its circuit breaker that +// reports response headers. +type observedFetcher interface { + FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) +} + +// fetchUpstream fetches downloadURL, reporting its Last-Modified when ctx +// asks for it. +func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { + report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) + observer, ok := p.Fetcher.(observedFetcher) + if report == nil || !ok { + return p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + } + observed, err := observer.FetchObservedWithHeaders(ctx, downloadURL, headers) + if err != nil { + return nil, err + } + if observed.Observation != nil { + if t, err := http.ParseTime(observed.Observation.Headers.Get(headerLastModified)); err == nil { + report(t) + } + } + return observed.Artifact, nil +} + // streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its // body to the caller without storing it. // diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4b677acb..c201e74b 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,6 +1,7 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -170,32 +171,48 @@ func (s mavenSource) cacheFilename(filename string) string { const mavenUpstreamHashBytes = 8 func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - cacheKey := source.metadataCacheKey(urlPath) + if h.cooldownEnabled() && h.serveCooldownMetadata(w, r, source, urlPath) { + return + } + + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) + if err != nil { + h.writeMetadataError(w, err) + return + } + + h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) +} + +// fetchMetadata fetches a metadata file from the source's upstream, or from +// its fallback when the upstream doesn't have it, and returns it with the +// key it is cached under. +func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { + cacheKey = source.metadataCacheKey(urlPath) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") } } - if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return body, contentType, cacheKey, err +} + +func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) return } - - h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } // handleDownload serves an artifact file, fetching and caching from upstream if needed. @@ -208,6 +225,14 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so return } + // The .pom of a version whose time isn't known yet carries it in its + // own Last-Modified, so it is checked once fetched, without a request + // of its own. Maven fetches the .pom before a version's other files. + ctx, checkAfter := h.cooldownFromDownload(r.Context(), source, urlPath, version) + if !checkAfter && h.checkCooldown(w, r, source, urlPath, version) { + return + } + // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) @@ -217,19 +242,23 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, pluginPortalURL) + result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } + if checkAfter && h.checkCooldown(w, r, source, urlPath, version) { + _ = result.Reader.Close() + return + } ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go new file mode 100644 index 00000000..d67d9c9c --- /dev/null +++ b/internal/handler/maven_cooldown.go @@ -0,0 +1,475 @@ +package handler + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "path" + "slices" + "strings" + "time" +) + +// Maven cooldown measures a version by the Last-Modified header of its .pom. +// On Maven Central that is the time the version was deployed. The header is +// read once per version and stored in the versions table, since release +// files don't change. +// +// maven-metadata.xml has a single for the whole artifact, so +// the proxy only looks at versions that can still be in cooldown: none when +// is older than the window, and otherwise the versions from +// the end of the list back to the first one that is old enough. Maven +// Central blocks clients that send bursts of requests, so the proxy doesn't +// read every version's time. +// +// SNAPSHOT versions are left alone: their files change after publishing. + +const errMavenCooldown = "version is in cooldown" + +// mavenTimeUserAgent is the User-Agent of publish-time requests, the one +// the artifact fetcher sends. +const mavenTimeUserAgent = "git-pkgs-proxy/1.0" + +const ( + mavenMetadataFile = "maven-metadata.xml" + mavenSnapshot = "-SNAPSHOT" + // mavenLastUpdatedLayout is the UTC timestamp format of . + mavenLastUpdatedLayout = "20060102150405" +) + +// mavenChecksums are the checksum files served beside maven-metadata.xml. +// With cooldown on they are computed from the metadata as served. +var mavenChecksums = map[string]func() hash.Hash{ + ".md5": md5.New, + ".sha1": sha1.New, + ".sha256": sha256.New, + ".sha512": sha512.New, +} + +func (h *MavenHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func mavenPackagePURL(group, artifact string) string { + return canonicalPackagePURL("maven", group+":"+artifact) +} + +// mavenCoordinates returns the group and artifact of an artifact directory +// such as com/google/guava/guava. +func mavenCoordinates(artifactDir string) (group, artifact string, ok bool) { + i := strings.LastIndex(artifactDir, "/") + if i <= 0 || i == len(artifactDir)-1 { + return "", "", false + } + return strings.ReplaceAll(artifactDir[:i], "/", "."), artifactDir[i+1:], true +} + +// usableMavenVersion reports whether version can be put in a URL path as +// one segment. Versions come from upstream metadata. +func usableMavenVersion(version string) bool { + return version != "" && version != "." && version != ".." && !strings.ContainsAny(version, "/\\?#%") +} + +// mavenTimeKey returns the versions-table PURLs a version's time is stored +// under. A named repository's keys carry its upstream as repository_url: +// the same coordinates there can have been published at another time than +// in the default repository. +func (h *MavenHandler) mavenTimeKey(source mavenSource, group, artifact, version string) (versionPURL, packagePURL string, ok bool) { + if h.proxy.DB == nil { + return "", "", false + } + name := group + ":" + artifact + versionPURL, packagePURL = canonicalVersionPURL("maven", name, version), canonicalPackagePURL("maven", name) + if source.name != "" { + qualifier := "?repository_url=" + url.QueryEscape(source.upstreamURL) + versionPURL += qualifier + packagePURL += qualifier + } + return versionPURL, packagePURL, true +} + +// publishTime returns a version's publish time from the versions table, or +// from the Last-Modified of its .pom, which is then stored. A version +// without a .pom, or whose .pom has no Last-Modified, gets the zero time and +// is let through, as in the other ecosystems. A failed request is an error, +// so a rate-limited upstream can't let a version through. +func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, artifactDir, version string) (time.Time, error) { + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || !usableMavenVersion(version) { + return time.Time{}, nil + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if store { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ver.PublishedAt.Time, nil + } + } + + pomPath := artifactDir + "/" + version + "/" + artifact + "-" + version + ".pom" + published, found, err := h.pomLastModified(ctx, source.upstreamURL+"/"+pomPath) + if err == nil && !found && source.fallbackURL != "" { + published, _, err = h.pomLastModified(ctx, source.fallbackURL+"/"+pomPath) + } + if err != nil { + return time.Time{}, err + } + if store && !published.IsZero() { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + } + return published, nil +} + +// cooldownFromDownload returns a context that stores the publish time of a +// version from the download of its .pom or Gradle .module, and whether the +// cooldown check is to run after the download. It does so only for those +// files of a version whose time is not stored yet; for anything else the +// check runs before. Both files are deployed with the version, and +// whichever a client fetches first gives its time. +func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return ctx, false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if base := artifact + "-" + version; !ok || !usableMavenVersion(version) || + (path.Base(urlPath) != base+".pom" && path.Base(urlPath) != base+".module") { + return ctx, false + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if !store { + return ctx, false + } + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ctx, false + } + return WithUpstreamLastModified(ctx, func(published time.Time) { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + }), true +} + +// pomLastModified sends a HEAD request for a .pom and returns its +// Last-Modified. found is false when upstream doesn't have the file. Only a +// 2xx response's header is used: an error page's date says nothing about the +// version. +func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (published time.Time, found bool, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, pomURL, nil) + if err != nil { + return time.Time{}, false, err + } + // Name the proxy as its artifact downloads do; Maven Central asks + // tools not to send an HTTP library's default. + req.Header.Set("User-Agent", mavenTimeUserAgent) + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) + } + _ = resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return time.Time{}, false, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + // The gate has just noted the upstream's Retry-After; pass it on. + if err := h.proxy.RateLimits.Held(pomURL); err != nil { + return time.Time{}, false, err + } + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) + case resp.StatusCode < 200 || resp.StatusCode > 299: + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) + } + published, err = http.ParseTime(resp.Header.Get(headerLastModified)) + if err != nil { + return time.Time{}, true, nil + } + return published, true, nil +} + +// checkCooldown answers a request for a file of a version in cooldown, or +// whose time couldn't be read, and reports whether it did. The answer is +// 403, as for a version on the denylist. +func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath, version string) bool { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || h.proxy.Cooldown.For("maven", mavenPackagePURL(group, artifact)) == 0 { + return false + } + published, err := h.publishTime(r.Context(), source, artifactDir, version) + if err != nil { + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not read maven version time", + "group", group, "artifact", artifact, "version", version, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + h.proxy.Logger.Info("cooldown: withholding maven version", + "group", group, "artifact", artifact, "version", version) + http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) + return true + } + return false +} + +// serveCooldownMetadata serves maven-metadata.xml without versions in +// cooldown, and its checksum files computed from what is served. It reports +// false for other files beside the metadata, such as a signature, when the +// metadata is served unchanged, so they are proxied as usual. +func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) bool { + filename := path.Base(urlPath) + suffix, ok := strings.CutPrefix(filename, mavenMetadataFile) + if !ok { + return false + } + metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) + if err != nil { + h.writeMetadataError(w, err) + return true + } + + m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { + // The rewrite is shared by requests waiting on it and cached, so it + // finishes even if the request that started it goes away. + out, err := h.filterMetadata(context.WithoutCancel(r.Context()), source, path.Dir(urlPath), in) + return out, nil, err + }) + if err != nil { + if r.Context().Err() != nil { + return true + } + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not filter maven metadata", "path", metadataPath, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + + switch { + case suffix == "": + // The metadata cache records JSON when an upstream sent no type, + // and maven-metadata.xml is always XML. + if contentType == "" || contentType == contentTypeJSON { + contentType = "text/xml" + } + markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) + serveRewrittenMetadataAs(w, r, m, contentType) + case mavenChecksums[suffix] != nil: + sum := mavenChecksums[suffix]() + _, _ = sum.Write(m.body) + w.Header().Set(headerContentType, "text/plain") + _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) + case bytes.Equal(m.body, body): + return false + default: + // A signature of the upstream metadata doesn't match what is served. + http.Error(w, "not found", http.StatusNotFound) + } + return true +} + +// filterMetadata removes versions in cooldown from an artifact's +// maven-metadata.xml and points and at the newest +// versions left. Metadata without a version list, such as a group's plugin +// list or a SNAPSHOT's build list, is returned unchanged. The document is +// edited in place rather than encoded again, so everything else in it stays +// as upstream sent it. +func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, artifactDir string, body []byte) ([]byte, error) { + doc, err := parseMavenMetadata(body) + if err != nil { + return nil, err + } + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || len(doc.versions) == 0 { + return body, nil + } + packagePURL := mavenPackagePURL(group, artifact) + // No version can be newer than the metadata listing it. + if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && + h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + return body, nil + } + + removed := map[string]bool{} + for i := len(doc.versions) - 1; i >= 0; i-- { + v := doc.versions[i].value + if strings.HasSuffix(v, mavenSnapshot) { + continue + } + published, err := h.publishTime(ctx, source, artifactDir, v) + if err != nil { + return nil, fmt.Errorf("reading time of %s: %w", v, err) + } + if published.IsZero() { + continue + } + if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + break + } + h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) + removed[v] = true + } + if len(removed) == 0 { + return body, nil + } + return doc.without(body, removed), nil +} + +// mavenElement is an element of maven-metadata.xml: its text and where it +// starts and ends in the document. +type mavenElement struct { + value string + start, end int +} + +type mavenMetadata struct { + versions []mavenElement + latest, release *mavenElement + lastUpdated string +} + +// parseMavenMetadata reads the version list, , and +// of a maven-metadata.xml, with their offsets. +func parseMavenMetadata(body []byte) (*mavenMetadata, error) { + dec := xml.NewDecoder(bytes.NewReader(body)) + // Versions are ASCII, so any declared charset can be read as bytes. + dec.CharsetReader = func(_ string, in io.Reader) (io.Reader, error) { return in, nil } + + doc := &mavenMetadata{} + var stack []string + var current *mavenElement + for { + offset := int(dec.InputOffset()) + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parsing maven metadata: %w", err) + } + switch t := tok.(type) { + case xml.StartElement: + stack = append(stack, t.Name.Local) + switch strings.Join(stack, "/") { + case "metadata/versioning/versions/version", + "metadata/versioning/latest", + "metadata/versioning/release", + "metadata/versioning/lastUpdated": + current = &mavenElement{start: offset} + } + case xml.CharData: + if current != nil { + current.value += string(t) + } + case xml.EndElement: + if current != nil { + current.value = strings.TrimSpace(current.value) + current.end = int(dec.InputOffset()) + switch t.Name.Local { + case "version": + doc.versions = append(doc.versions, *current) + case "latest": + doc.latest = current + case "release": + doc.release = current + case "lastUpdated": + doc.lastUpdated = current.value + } + current = nil + } + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + } + } + return doc, nil +} + +// without returns body with the removed versions taken out of the version +// list, and and set to the last versions left in list +// order, which is how the repository chose them. Either is dropped when no +// version is left for it. +func (doc *mavenMetadata) without(body []byte, removed map[string]bool) []byte { + type edit struct { + start, end int + text string + } + var edits []edit + drop := func(e mavenElement) { + edits = append(edits, edit{start: lineStart(body, e.start), end: e.end}) + } + var latest, release string + for _, v := range doc.versions { + if removed[v.value] { + drop(v) + continue + } + latest = v.value + if !strings.HasSuffix(v.value, mavenSnapshot) { + release = v.value + } + } + for _, field := range []struct { + el *mavenElement + name string + next string + }{{doc.latest, "latest", latest}, {doc.release, "release", release}} { + if field.el == nil || !removed[field.el.value] { + continue + } + if field.next == "" { + drop(*field.el) + continue + } + edits = append(edits, edit{field.el.start, field.el.end, "<" + field.name + ">" + field.next + ""}) + } + + // Edits don't overlap; apply them in document order. + slices.SortFunc(edits, func(a, b edit) int { return a.start - b.start }) + out := make([]byte, 0, len(body)) + pos := 0 + for _, e := range edits { + out = append(out, body[pos:e.start]...) + out = append(out, e.text...) + pos = e.end + } + return append(out, body[pos:]...) +} + +// lineStart moves an element's start back over the indentation and line +// break before it, so removing the element removes its line. +func lineStart(body []byte, start int) int { + i := start + for i > 0 && (body[i-1] == ' ' || body[i-1] == '\t') { + i-- + } + if i > 0 && body[i-1] == '\n' { + i-- + if i > 0 && body[i-1] == '\r' { + i-- + } + return i + } + return start +} diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go new file mode 100644 index 00000000..828d1ffa --- /dev/null +++ b/internal/handler/maven_cooldown_test.go @@ -0,0 +1,530 @@ +package handler + +import ( + "compress/gzip" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "fmt" + "hash" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" +) + +const mavenTestDir = "/com/example/lib/" + +// mavenCooldownUpstream serves com.example:lib. Each version's .pom has the +// Last-Modified in times; pomStatus overrides the status of a version's +// .pom. Requests for .pom files are counted. +type mavenCooldownUpstream struct { + server *httptest.Server + versions []string + latest string + release string + lastUpdated time.Time + times map[string]time.Time + pomStatus map[string]int + + mu sync.Mutex + poms map[string]int + heads int + userAgents []string +} + +func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { + t.Helper() + now := time.Now() + u := &mavenCooldownUpstream{ + versions: []string{"1.0", "1.1", "2.0", "2.1", "3.0-SNAPSHOT"}, + latest: "3.0-SNAPSHOT", + release: "2.1", + lastUpdated: now.Add(-time.Hour), + times: map[string]time.Time{ + "1.0": now.Add(-30 * 24 * time.Hour), + "1.1": now.Add(-10 * 24 * time.Hour), + "2.0": now.Add(-2 * time.Hour), + "2.1": now.Add(-time.Hour), + "3.0-SNAPSHOT": now.Add(-time.Minute), + }, + pomStatus: map[string]int{}, + poms: map[string]int{}, + } + u.server = httptest.NewServer(http.HandlerFunc(u.serve)) + t.Cleanup(u.server.Close) + return u +} + +// serve answers for the default upstream and, under /portal, for the +// Gradle Plugin Portal fallback alike. +func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { + rest, ok := strings.CutPrefix(strings.TrimPrefix(r.URL.Path, "/portal"), mavenTestDir) + if !ok { + http.NotFound(w, r) + return + } + switch rest { + case "maven-metadata.xml": + _, _ = w.Write([]byte(u.metadata())) + return + case "maven-metadata.xml.asc": + _, _ = w.Write([]byte("signature")) + return + } + version, file, _ := strings.Cut(rest, "/") + published, known := u.times[version] + if !known { + http.NotFound(w, r) + return + } + if strings.HasSuffix(file, ".module") { + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + } + if strings.HasSuffix(file, ".pom") { + u.mu.Lock() + u.poms[version]++ + if r.Method == http.MethodHead { + u.heads++ + u.userAgents = append(u.userAgents, r.UserAgent()) + } + u.mu.Unlock() + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + if status := u.pomStatus[version]; status != 0 { + // An error page dated long ago must not count as an old version. + w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) + if status == http.StatusTooManyRequests { + w.Header().Set("Retry-After", "60") + } + w.WriteHeader(status) + return + } + } + _, _ = w.Write([]byte(file + " of " + version)) +} + +func (u *mavenCooldownUpstream) metadata() string { + var b strings.Builder + b.WriteString("\n\n com.example\n lib\n \n") + b.WriteString(" " + u.latest + "\n " + u.release + "\n \n") + for _, v := range u.versions { + b.WriteString(" " + v + "\n") + } + b.WriteString(" \n " + u.lastUpdated.UTC().Format(mavenLastUpdatedLayout) + "\n \n\n") + return b.String() +} + +// pomRequests returns how many requests each version's .pom got. +func (u *mavenCooldownUpstream) pomRequests() map[string]int { + u.mu.Lock() + defer u.mu.Unlock() + out := make(map[string]int, len(u.poms)) + for v, n := range u.poms { + out[v] = n + } + return out +} + +func (u *mavenCooldownUpstream) totalPOMRequests() int { + total := 0 + for _, n := range u.pomRequests() { + total += n + } + return total +} + +func newMavenCooldownHandler(t *testing.T, u *mavenCooldownUpstream) (http.Handler, *Proxy) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = u.server.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.server.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal").Routes(), proxy +} + +func mavenMetadataVersions(t *testing.T, body string) []string { + t.Helper() + doc, err := parseMavenMetadata([]byte(body)) + if err != nil { + t.Fatalf("served metadata does not parse: %v\n%s", err, body) + } + var versions []string + for _, v := range doc.versions { + versions = append(versions, v.value) + } + return versions +} + +func TestMavenCooldownFiltersMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + body := w.Body.String() + if got := strings.Join(mavenMetadataVersions(t, body), " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + if !strings.Contains(body, "1.1") || !strings.Contains(body, "3.0-SNAPSHOT") { + t.Errorf("release or latest not recomputed:\n%s", body) + } + if strings.Contains(body, "\n\n") { + t.Errorf("removed versions left blank lines:\n%s", body) + } + + // The walk stops at 1.1, the first version old enough, and skips the + // SNAPSHOT. + want := map[string]int{"2.1": 1, "2.0": 1, "1.1": 1} + if got := u.pomRequests(); len(got) != len(want) || got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 { + t.Errorf("pom requests = %v, want %v", got, want) + } + + // Times are stored, so the next request makes no .pom requests. + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("time of 2.1 not stored: %v %v", ver, err) + } + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != body { + t.Errorf("second response differs:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests after second metadata request = %d, want 3", n) + } +} + +func TestMavenCooldownRecomputesLatest(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"1.0", "1.1", "2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if !strings.Contains(body, "1.1") || !strings.Contains(body, "1.1") { + t.Errorf("latest and release should be 1.1:\n%s", body) + } +} + +func TestMavenCooldownDropsLatestWhenNothingIsLeft(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if strings.Contains(body, "") || strings.Contains(body, "") || strings.Contains(body, "") { + t.Errorf("no version is out of cooldown:\n%s", body) + } + if !strings.Contains(body, "\n ") { + t.Errorf("version list not emptied in place:\n%s", body) + } +} + +func TestMavenCooldownChecksumsMatchMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.Bytes() + for suffix, newHash := range map[string]func() hash.Hash{ + ".md5": md5.New, ".sha1": sha1.New, ".sha256": sha256.New, ".sha512": sha512.New, + } { + sum := newHash() + _, _ = sum.Write(body) + w := getMaven(h, mavenTestDir+"maven-metadata.xml"+suffix) + if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum.Sum(nil)) { + t.Errorf("%s: status = %d body %q, want checksum of served metadata", suffix, w.Code, w.Body) + } + } + // A signature of the upstream metadata would not match. + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusNotFound { + t.Errorf(".asc status = %d, want 404", w.Code) + } + + // Unchanged metadata keeps its signature. + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusOK || w.Body.String() != "signature" { + t.Errorf("unchanged .asc: status = %d body %q, want upstream's", w.Code, w.Body) + } +} + +func TestMavenCooldownRefusesFiles(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + for _, file := range []string{"2.1/lib-2.1.pom", "2.1/lib-2.1.jar", "2.1/lib-2.1.jar.sha1", "2.0/lib-2.0-sources.jar"} { + w := getMaven(h, mavenTestDir+file) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errMavenCooldown) { + t.Errorf("%s: status = %d body %q, want 403 cooldown", file, w.Code, w.Body) + } + } + for _, file := range []string{"1.1/lib-1.1.jar", "3.0-SNAPSHOT/lib-3.0-SNAPSHOT.jar"} { + if w := getMaven(h, mavenTestDir+file); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200", file, w.Code) + } + } + // One .pom request per version, none for the SNAPSHOT. + if got := u.pomRequests(); got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 || got["3.0-SNAPSHOT"] != 0 { + t.Errorf("pom requests = %v", got) + } +} + +// TestMavenCooldownPOMDownloadCarriesItsTime checks that a build fetching +// pinned versions, which never reads maven-metadata.xml, costs no requests +// beyond its downloads: the .pom's own Last-Modified gives the version's +// time, and Maven fetches the .pom before the version's other files. +func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + for _, c := range []struct { + file string + want int + }{ + {"2.1/lib-2.1.pom", http.StatusForbidden}, + {"2.1/lib-2.1.jar", http.StatusForbidden}, + {"1.0/lib-1.0.pom", http.StatusOK}, + {"1.0/lib-1.0.jar", http.StatusOK}, + } { + if w := getMaven(h, mavenTestDir+c.file); w.Code != c.want { + t.Errorf("%s: status = %d, want %d", c.file, w.Code, c.want) + } + } + u.mu.Lock() + heads := u.heads + u.mu.Unlock() + if heads != 0 { + t.Errorf("HEAD requests = %d, want 0", heads) + } + if got := u.pomRequests(); got["2.1"] != 1 || got["1.0"] != 1 { + t.Errorf("pom requests = %v, want one GET each", got) + } + ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(u.times["2.1"].Truncate(time.Second)) { + t.Errorf("stored time of 2.1 = %v, want %v", ver, u.times["2.1"]) + } +} + +// TestMavenCooldownModuleDownloadCarriesItsTime checks that a Gradle +// .module fetched before the .pom gives the version's time as the .pom +// would. +func TestMavenCooldownModuleDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.module"); w.Code != http.StatusForbidden { + t.Errorf("2.1 .module: status = %d, want 403", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.module"); w.Code != http.StatusOK { + t.Errorf("1.0 .module: status = %d, want 200", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.jar"); w.Code != http.StatusOK { + t.Errorf("1.0 .jar: status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +// TestMavenCooldownTimeRequestsNameTheProxy checks the User-Agent of the +// HEAD requests for publish times: Maven Central asks tools to name +// themselves rather than send an HTTP library's default. +func TestMavenCooldownTimeRequestsNameTheProxy(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + getMaven(h, mavenTestDir+"maven-metadata.xml") + u.mu.Lock() + defer u.mu.Unlock() + if len(u.userAgents) == 0 { + t.Fatal("no HEAD requests") + } + for _, ua := range u.userAgents { + if ua != mavenTimeUserAgent { + t.Errorf("User-Agent = %q, want %q", ua, mavenTimeUserAgent) + } + } +} + +// TestMavenCooldownMetadataServedLikeOtherRewrites checks that filtered +// metadata is gzip-compressed for clients that accept it, as the other +// ecosystems' rewritten metadata is. +func TestMavenCooldownMetadataServedLikeOtherRewrites(t *testing.T) { + u := newMavenCooldownUpstream(t) + // Enough versions that the document compresses. + for i := range 50 { + v := fmt.Sprintf("0.%d", i) + u.versions = append([]string{v}, u.versions...) + u.times[v] = time.Now().Add(-100 * 24 * time.Hour) + } + h, proxy := newMavenCooldownHandler(t, u) + // The server always sets up the rewrite cache, which keeps the gzip copy. + proxy.SetMetadataRewriteCacheSize(1 << 20) + + req := httptest.NewRequest(http.MethodGet, mavenTestDir+"maven-metadata.xml", nil) + req.Header.Set(headerAcceptEncoding, "gzip") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "gzip" { + t.Fatalf("status = %d, Content-Encoding %q; want gzip", w.Code, w.Header().Get(headerContentEncoding)) + } + if ct := w.Header().Get(headerContentType); !strings.Contains(ct, "xml") { + t.Errorf("Content-Type = %q, want XML", ct) + } + zr, err := gzip.NewReader(w.Body) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(zr) + if versions := mavenMetadataVersions(t, string(body)); slices.Contains(versions, "2.1") { + t.Errorf("2.1 listed in the gzip body: %v", versions) + } +} + +func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + // The times would filter 2.0 and 2.1 if they were read. + h, _ := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.1"] = http.StatusNotFound + h, proxy := newMavenCooldownHandler(t, u) + + // A missing .pom gives no time: 2.1 stays listed, and the walk goes on + // to 2.0 rather than stopping at the error page's old date. + versions := mavenMetadataVersions(t, getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 2.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 2.1 3.0-SNAPSHOT", got) + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 404 stored: %v", ver.PublishedAt.Time) + } + + // Any other error fails closed. + u.pomStatus["2.1"] = http.StatusTooManyRequests + for _, p := range []string{"maven-metadata.xml", "maven-metadata.xml.sha1", "2.1/lib-2.1.jar"} { + if w := getMaven(h, mavenTestDir+p); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", p, w.Code) + } + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 429 stored: %v", ver.PublishedAt.Time) + } +} + +// TestMavenCooldownRateLimitedPOMIsHeld checks that a 429 on a .pom time +// request is answered like any held upstream request, with 503 and the +// upstream's Retry-After, so clients wait instead of failing the build. +func TestMavenCooldownRateLimitedPOMIsHeld(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.0"] = http.StatusTooManyRequests + h, proxy := newMavenCooldownHandler(t, u) + gate := registryclient.NewRateLimitGate(u.server.Client().Transport) + proxy.HTTPClient = &http.Client{Transport: gate} + proxy.RateLimits = gate + + // Not the .pom, so its time is read with a HEAD. + w := getMaven(h, mavenTestDir+"2.0/lib-2.0.jar") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } +} + +func TestMavenCooldownDisabledPassesThrough(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = nil + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownPackageOverride(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:maven/com.example/lib": "0"}} + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownNamedRepositoryStoresItsOwnTimes(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + // The named repository is the same server under /portal, which serves + // the same files. + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + + for range 2 { + versions := mavenMetadataVersions(t, getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Fatalf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + } + if w := getMaven(h, "/mirror"+mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusForbidden { + t.Errorf("jar status = %d, want 403", w.Code) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests = %d, want 3", n) + } + + // The default repository doesn't use the named one's times. + _ = getMaven(h, mavenTestDir+"maven-metadata.xml") + if n := u.totalPOMRequests(); n != 6 { + t.Errorf("pom requests after default metadata = %d, want 6", n) + } +} + +func TestMavenMetadataWithoutCompact(t *testing.T) { + in := `2212` + doc, err := parseMavenMetadata([]byte(in)) + if err != nil { + t.Fatal(err) + } + got := string(doc.without([]byte(in), map[string]bool{"2": true})) + want := `111` + if got != want { + t.Errorf("got %s\nwant %s", got, want) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 0569a9cd..991443af 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -402,6 +402,12 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri // accepting gzip gets the compressed copy when there is one. Being another // representation, it has an ETag of its own. func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { + serveRewrittenMetadataAs(w, r, m, contentTypeJSON) +} + +// serveRewrittenMetadataAs is serveRewrittenMetadata for a document of +// another content type. +func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { body, etag := m.body, m.etag compressed := m.gzipped != nil && acceptsGzip(r) if compressed { @@ -416,7 +422,7 @@ func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenM w.WriteHeader(http.StatusNotModified) return } - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") }