From 562faac436f4979c205ea6a75b57efd230697a06 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 1/6] Split Maven metadata fetch from its response Cooldown needs to read maven-metadata.xml before serving it, with the same Gradle Plugin Portal fallback and error handling. --- internal/handler/maven.go | 43 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 4b677acb..44bd6e71 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -1,6 +1,7 @@ package handler import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -170,32 +171,44 @@ func (s mavenSource) cacheFilename(filename string) string { const mavenUpstreamHashBytes = 8 func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { - cacheKey := source.metadataCacheKey(urlPath) + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) + if err != nil { + h.writeMetadataError(w, err) + return + } + + h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) +} + +// fetchMetadata fetches a metadata file from the source's upstream, or from +// its fallback when the upstream doesn't have it, and returns it with the +// key it is cached under. +func (h *MavenHandler) fetchMetadata(ctx context.Context, source mavenSource, urlPath string) (body []byte, contentType, cacheKey string, err error) { + cacheKey = source.metadataCacheKey(urlPath) upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) - body, contentType, err := h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, upstreamURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, upstreamURL, "*/*") if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven metadata unavailable in primary upstream, trying Gradle Plugin Portal", "path", urlPath) - body, contentType, err = h.proxy.FetchOrCacheMetadata(r.Context(), "maven", cacheKey, pluginPortalURL, "*/*") + body, contentType, err = h.proxy.FetchOrCacheMetadata(ctx, "maven", cacheKey, pluginPortalURL, "*/*") } } - if err != nil { - if errors.Is(err, ErrUpstreamNotFound) { - http.Error(w, "not found", http.StatusNotFound) - return - } - if writeUpstreamHeld(w, err) { - return - } - h.proxy.Logger.Error("metadata fetch failed", "error", err) - http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) + return body, contentType, cacheKey, err +} + +func (h *MavenHandler) writeMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + http.Error(w, "not found", http.StatusNotFound) return } - - h.proxy.writeMetadataCachedResponse(w, r, "maven", cacheKey, body, contentType) + if writeUpstreamHeld(w, err) { + return + } + h.proxy.Logger.Error("metadata fetch failed", "error", err) + http.Error(w, "failed to fetch from upstream", http.StatusBadGateway) } // handleDownload serves an artifact file, fetching and caching from upstream if needed. From 74f8eb378b1597254183da9467379d9a815395f3 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 2/6] Apply cooldown to Maven A version's publish time is the Last-Modified of its .pom, read with a HEAD request once per version and stored in the versions table. Named repositories store theirs under a repository_url qualifier. Versions in cooldown are left out of maven-metadata.xml, and are moved to the last versions left, and the metadata's checksum files are computed from what is served. Files of a version in cooldown get 403. Only versions that can still be in cooldown are checked: none when is older than the window, otherwise from the end of the list back to the first old enough version. Part of #132. --- internal/handler/maven.go | 8 + internal/handler/maven_cooldown.go | 443 ++++++++++++++++++++++++ internal/handler/maven_cooldown_test.go | 383 ++++++++++++++++++++ 3 files changed, 834 insertions(+) create mode 100644 internal/handler/maven_cooldown.go create mode 100644 internal/handler/maven_cooldown_test.go diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 44bd6e71..48859065 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -171,6 +171,10 @@ func (s mavenSource) cacheFilename(filename string) string { const mavenUpstreamHashBytes = 8 func (h *MavenHandler) handleMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) { + if h.cooldownEnabled() && h.serveCooldownMetadata(w, r, source, urlPath) { + return + } + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, urlPath) if err != nil { h.writeMetadataError(w, err) @@ -221,6 +225,10 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so return } + if h.checkCooldown(w, r, source, urlPath, version) { + return + } + // Maven uses group:artifact as the package name name := fmt.Sprintf("%s:%s", group, artifact) diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go new file mode 100644 index 00000000..ca5a03a0 --- /dev/null +++ b/internal/handler/maven_cooldown.go @@ -0,0 +1,443 @@ +package handler + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "path" + "slices" + "strings" + "time" +) + +// Maven cooldown measures a version by the Last-Modified header of its .pom. +// On Maven Central that is the time the version was deployed. The header is +// read once per version and stored in the versions table, since release +// files don't change. +// +// maven-metadata.xml has a single for the whole artifact, so +// the proxy only looks at versions that can still be in cooldown: none when +// is older than the window, and otherwise the versions from +// the end of the list back to the first one that is old enough. Maven +// Central blocks clients that send bursts of requests, so the proxy doesn't +// read every version's time. +// +// SNAPSHOT versions are left alone: their files change after publishing. + +const errMavenCooldown = "version is in cooldown" + +const ( + mavenMetadataFile = "maven-metadata.xml" + mavenSnapshot = "-SNAPSHOT" + // mavenLastUpdatedLayout is the UTC timestamp format of . + mavenLastUpdatedLayout = "20060102150405" +) + +// mavenChecksums are the checksum files served beside maven-metadata.xml. +// With cooldown on they are computed from the metadata as served. +var mavenChecksums = map[string]func() hash.Hash{ + ".md5": md5.New, + ".sha1": sha1.New, + ".sha256": sha256.New, + ".sha512": sha512.New, +} + +func (h *MavenHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func mavenPackagePURL(group, artifact string) string { + return canonicalPackagePURL("maven", group+":"+artifact) +} + +// mavenCoordinates returns the group and artifact of an artifact directory +// such as com/google/guava/guava. +func mavenCoordinates(artifactDir string) (group, artifact string, ok bool) { + i := strings.LastIndex(artifactDir, "/") + if i <= 0 || i == len(artifactDir)-1 { + return "", "", false + } + return strings.ReplaceAll(artifactDir[:i], "/", "."), artifactDir[i+1:], true +} + +// usableMavenVersion reports whether version can be put in a URL path as +// one segment. Versions come from upstream metadata. +func usableMavenVersion(version string) bool { + return version != "" && version != "." && version != ".." && !strings.ContainsAny(version, "/\\?#%") +} + +// mavenTimeKey returns the versions-table PURLs a version's time is stored +// under. A named repository's keys carry its upstream as repository_url: +// the same coordinates there can have been published at another time than +// in the default repository. +func (h *MavenHandler) mavenTimeKey(source mavenSource, group, artifact, version string) (versionPURL, packagePURL string, ok bool) { + if h.proxy.DB == nil { + return "", "", false + } + name := group + ":" + artifact + versionPURL, packagePURL = canonicalVersionPURL("maven", name, version), canonicalPackagePURL("maven", name) + if source.name != "" { + qualifier := "?repository_url=" + url.QueryEscape(source.upstreamURL) + versionPURL += qualifier + packagePURL += qualifier + } + return versionPURL, packagePURL, true +} + +// publishTime returns a version's publish time from the versions table, or +// from the Last-Modified of its .pom, which is then stored. A version +// without a .pom, or whose .pom has no Last-Modified, gets the zero time and +// is let through, as in the other ecosystems. A failed request is an error, +// so a rate-limited upstream can't let a version through. +func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, artifactDir, version string) (time.Time, error) { + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || !usableMavenVersion(version) { + return time.Time{}, nil + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if store { + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ver.PublishedAt.Time, nil + } + } + + pomPath := artifactDir + "/" + version + "/" + artifact + "-" + version + ".pom" + published, found, err := h.pomLastModified(ctx, source.upstreamURL+"/"+pomPath) + if err == nil && !found && source.fallbackURL != "" { + published, _, err = h.pomLastModified(ctx, source.fallbackURL+"/"+pomPath) + } + if err != nil { + return time.Time{}, err + } + if store && !published.IsZero() { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + } + return published, nil +} + +// pomLastModified sends a HEAD request for a .pom and returns its +// Last-Modified. found is false when upstream doesn't have the file. Only a +// 2xx response's header is used: an error page's date says nothing about the +// version. +func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (published time.Time, found bool, err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, pomURL, nil) + if err != nil { + return time.Time{}, false, err + } + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) + } + _ = resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: + return time.Time{}, false, nil + case resp.StatusCode < 200 || resp.StatusCode > 299: + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) + } + published, err = http.ParseTime(resp.Header.Get(headerLastModified)) + if err != nil { + return time.Time{}, true, nil + } + return published, true, nil +} + +// checkCooldown answers a request for a file of a version in cooldown, or +// whose time couldn't be read, and reports whether it did. The answer is +// 403, as for a version on the denylist. +func (h *MavenHandler) checkCooldown(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath, version string) bool { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || h.proxy.Cooldown.For("maven", mavenPackagePURL(group, artifact)) == 0 { + return false + } + published, err := h.publishTime(r.Context(), source, artifactDir, version) + if err != nil { + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not read maven version time", + "group", group, "artifact", artifact, "version", version, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + if !h.proxy.Cooldown.IsAllowed("maven", mavenPackagePURL(group, artifact), published) { + h.proxy.Logger.Info("cooldown: withholding maven version", + "group", group, "artifact", artifact, "version", version) + http.Error(w, errMavenCooldown+": "+canonicalVersionPURL("maven", group+":"+artifact, version), http.StatusForbidden) + return true + } + return false +} + +// serveCooldownMetadata serves maven-metadata.xml without versions in +// cooldown, and its checksum files computed from what is served. It reports +// false for other files beside the metadata, such as a signature, when the +// metadata is served unchanged, so they are proxied as usual. +func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Request, source mavenSource, urlPath string) bool { + filename := path.Base(urlPath) + suffix, ok := strings.CutPrefix(filename, mavenMetadataFile) + if !ok { + return false + } + metadataPath := path.Join(path.Dir(urlPath), mavenMetadataFile) + body, contentType, cacheKey, err := h.fetchMetadata(r.Context(), source, metadataPath) + if err != nil { + h.writeMetadataError(w, err) + return true + } + + m, err := h.proxy.cachedRewrite(r.Context(), "maven", h.proxyURL, cacheKey, body, "", func(in []byte) ([]byte, []string, error) { + // The rewrite is shared by requests waiting on it and cached, so it + // finishes even if the request that started it goes away. + out, err := h.filterMetadata(context.WithoutCancel(r.Context()), source, path.Dir(urlPath), in) + return out, nil, err + }) + if err != nil { + if r.Context().Err() != nil { + return true + } + if writeUpstreamHeld(w, err) { + return true + } + h.proxy.Logger.Warn("cooldown: could not filter maven metadata", "path", metadataPath, "error", err) + http.Error(w, "failed to check cooldown", http.StatusBadGateway) + return true + } + + switch { + case suffix == "": + if contentType == "" || contentType == contentTypeJSON { + contentType = "text/xml" + } + serveMavenMetadata(w, r, m, contentType) + case mavenChecksums[suffix] != nil: + sum := mavenChecksums[suffix]() + _, _ = sum.Write(m.body) + w.Header().Set(headerContentType, "text/plain") + _, _ = io.WriteString(w, hex.EncodeToString(sum.Sum(nil))) + case bytes.Equal(m.body, body): + return false + default: + // A signature of the upstream metadata doesn't match what is served. + http.Error(w, "not found", http.StatusNotFound) + } + return true +} + +// serveMavenMetadata writes filtered metadata with its validators, or 304 +// when they show the client's copy is current. +func serveMavenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { + w.Header().Set(headerETag, m.etag) + if !m.lastModified.IsZero() { + w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) + } + if notModified(r, m.etag, m.lastModified) { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerContentType, contentType) + _, _ = w.Write(m.body) +} + +// filterMetadata removes versions in cooldown from an artifact's +// maven-metadata.xml and points and at the newest +// versions left. Metadata without a version list, such as a group's plugin +// list or a SNAPSHOT's build list, is returned unchanged. The document is +// edited in place rather than encoded again, so everything else in it stays +// as upstream sent it. +func (h *MavenHandler) filterMetadata(ctx context.Context, source mavenSource, artifactDir string, body []byte) ([]byte, error) { + doc, err := parseMavenMetadata(body) + if err != nil { + return nil, err + } + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || len(doc.versions) == 0 { + return body, nil + } + packagePURL := mavenPackagePURL(group, artifact) + // No version can be newer than the metadata listing it. + if lastUpdated, err := time.Parse(mavenLastUpdatedLayout, doc.lastUpdated); err == nil && + h.proxy.Cooldown.IsAllowed("maven", packagePURL, lastUpdated) { + return body, nil + } + + removed := map[string]bool{} + for i := len(doc.versions) - 1; i >= 0; i-- { + v := doc.versions[i].value + if strings.HasSuffix(v, mavenSnapshot) { + continue + } + published, err := h.publishTime(ctx, source, artifactDir, v) + if err != nil { + return nil, fmt.Errorf("reading time of %s: %w", v, err) + } + if published.IsZero() { + continue + } + if h.proxy.Cooldown.IsAllowed("maven", packagePURL, published) { + break + } + h.proxy.Logger.Info("cooldown: filtering maven version", "group", group, "artifact", artifact, "version", v) + removed[v] = true + } + if len(removed) == 0 { + return body, nil + } + return doc.without(body, removed), nil +} + +// mavenElement is an element of maven-metadata.xml: its text and where it +// starts and ends in the document. +type mavenElement struct { + value string + start, end int +} + +type mavenMetadata struct { + versions []mavenElement + latest, release *mavenElement + lastUpdated string +} + +// parseMavenMetadata reads the version list, , and +// of a maven-metadata.xml, with their offsets. +func parseMavenMetadata(body []byte) (*mavenMetadata, error) { + dec := xml.NewDecoder(bytes.NewReader(body)) + // Versions are ASCII, so any declared charset can be read as bytes. + dec.CharsetReader = func(_ string, in io.Reader) (io.Reader, error) { return in, nil } + + doc := &mavenMetadata{} + var stack []string + var current *mavenElement + for { + offset := int(dec.InputOffset()) + tok, err := dec.Token() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("parsing maven metadata: %w", err) + } + switch t := tok.(type) { + case xml.StartElement: + stack = append(stack, t.Name.Local) + switch strings.Join(stack, "/") { + case "metadata/versioning/versions/version", + "metadata/versioning/latest", + "metadata/versioning/release", + "metadata/versioning/lastUpdated": + current = &mavenElement{start: offset} + } + case xml.CharData: + if current != nil { + current.value += string(t) + } + case xml.EndElement: + if current != nil { + current.value = strings.TrimSpace(current.value) + current.end = int(dec.InputOffset()) + switch t.Name.Local { + case "version": + doc.versions = append(doc.versions, *current) + case "latest": + doc.latest = current + case "release": + doc.release = current + case "lastUpdated": + doc.lastUpdated = current.value + } + current = nil + } + if len(stack) > 0 { + stack = stack[:len(stack)-1] + } + } + } + return doc, nil +} + +// without returns body with the removed versions taken out of the version +// list, and and set to the last versions left in list +// order, which is how the repository chose them. Either is dropped when no +// version is left for it. +func (doc *mavenMetadata) without(body []byte, removed map[string]bool) []byte { + type edit struct { + start, end int + text string + } + var edits []edit + drop := func(e mavenElement) { + edits = append(edits, edit{start: lineStart(body, e.start), end: e.end}) + } + var latest, release string + for _, v := range doc.versions { + if removed[v.value] { + drop(v) + continue + } + latest = v.value + if !strings.HasSuffix(v.value, mavenSnapshot) { + release = v.value + } + } + for _, field := range []struct { + el *mavenElement + name string + next string + }{{doc.latest, "latest", latest}, {doc.release, "release", release}} { + if field.el == nil || !removed[field.el.value] { + continue + } + if field.next == "" { + drop(*field.el) + continue + } + edits = append(edits, edit{field.el.start, field.el.end, "<" + field.name + ">" + field.next + ""}) + } + + // Edits don't overlap; apply them in document order. + slices.SortFunc(edits, func(a, b edit) int { return a.start - b.start }) + out := make([]byte, 0, len(body)) + pos := 0 + for _, e := range edits { + out = append(out, body[pos:e.start]...) + out = append(out, e.text...) + pos = e.end + } + return append(out, body[pos:]...) +} + +// lineStart moves an element's start back over the indentation and line +// break before it, so removing the element removes its line. +func lineStart(body []byte, start int) int { + i := start + for i > 0 && (body[i-1] == ' ' || body[i-1] == '\t') { + i-- + } + if i > 0 && body[i-1] == '\n' { + i-- + if i > 0 && body[i-1] == '\r' { + i-- + } + return i + } + return start +} diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go new file mode 100644 index 00000000..74c42683 --- /dev/null +++ b/internal/handler/maven_cooldown_test.go @@ -0,0 +1,383 @@ +package handler + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +const mavenTestDir = "/com/example/lib/" + +// mavenCooldownUpstream serves com.example:lib. Each version's .pom has the +// Last-Modified in times; pomStatus overrides the status of a version's +// .pom. Requests for .pom files are counted. +type mavenCooldownUpstream struct { + server *httptest.Server + versions []string + latest string + release string + lastUpdated time.Time + times map[string]time.Time + pomStatus map[string]int + + mu sync.Mutex + poms map[string]int +} + +func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { + t.Helper() + now := time.Now() + u := &mavenCooldownUpstream{ + versions: []string{"1.0", "1.1", "2.0", "2.1", "3.0-SNAPSHOT"}, + latest: "3.0-SNAPSHOT", + release: "2.1", + lastUpdated: now.Add(-time.Hour), + times: map[string]time.Time{ + "1.0": now.Add(-30 * 24 * time.Hour), + "1.1": now.Add(-10 * 24 * time.Hour), + "2.0": now.Add(-2 * time.Hour), + "2.1": now.Add(-time.Hour), + "3.0-SNAPSHOT": now.Add(-time.Minute), + }, + pomStatus: map[string]int{}, + poms: map[string]int{}, + } + u.server = httptest.NewServer(http.HandlerFunc(u.serve)) + t.Cleanup(u.server.Close) + return u +} + +// serve answers for the default upstream and, under /portal, for the +// Gradle Plugin Portal fallback alike. +func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { + rest, ok := strings.CutPrefix(strings.TrimPrefix(r.URL.Path, "/portal"), mavenTestDir) + if !ok { + http.NotFound(w, r) + return + } + switch rest { + case "maven-metadata.xml": + _, _ = w.Write([]byte(u.metadata())) + return + case "maven-metadata.xml.asc": + _, _ = w.Write([]byte("signature")) + return + } + version, file, _ := strings.Cut(rest, "/") + published, known := u.times[version] + if !known { + http.NotFound(w, r) + return + } + if strings.HasSuffix(file, ".pom") { + u.mu.Lock() + u.poms[version]++ + u.mu.Unlock() + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + if status := u.pomStatus[version]; status != 0 { + // An error page dated long ago must not count as an old version. + w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) + w.WriteHeader(status) + return + } + } + _, _ = w.Write([]byte(file + " of " + version)) +} + +func (u *mavenCooldownUpstream) metadata() string { + var b strings.Builder + b.WriteString("\n\n com.example\n lib\n \n") + b.WriteString(" " + u.latest + "\n " + u.release + "\n \n") + for _, v := range u.versions { + b.WriteString(" " + v + "\n") + } + b.WriteString(" \n " + u.lastUpdated.UTC().Format(mavenLastUpdatedLayout) + "\n \n\n") + return b.String() +} + +// pomRequests returns how many requests each version's .pom got. +func (u *mavenCooldownUpstream) pomRequests() map[string]int { + u.mu.Lock() + defer u.mu.Unlock() + out := make(map[string]int, len(u.poms)) + for v, n := range u.poms { + out[v] = n + } + return out +} + +func (u *mavenCooldownUpstream) totalPOMRequests() int { + total := 0 + for _, n := range u.pomRequests() { + total += n + } + return total +} + +func newMavenCooldownHandler(t *testing.T, u *mavenCooldownUpstream) (http.Handler, *Proxy) { + t.Helper() + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true + proxy.HTTPClient = u.server.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(u.server.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + return NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal").Routes(), proxy +} + +func mavenMetadataVersions(t *testing.T, body string) []string { + t.Helper() + doc, err := parseMavenMetadata([]byte(body)) + if err != nil { + t.Fatalf("served metadata does not parse: %v\n%s", err, body) + } + var versions []string + for _, v := range doc.versions { + versions = append(versions, v.value) + } + return versions +} + +func TestMavenCooldownFiltersMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body) + } + body := w.Body.String() + if got := strings.Join(mavenMetadataVersions(t, body), " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + if !strings.Contains(body, "1.1") || !strings.Contains(body, "3.0-SNAPSHOT") { + t.Errorf("release or latest not recomputed:\n%s", body) + } + if strings.Contains(body, "\n\n") { + t.Errorf("removed versions left blank lines:\n%s", body) + } + + // The walk stops at 1.1, the first version old enough, and skips the + // SNAPSHOT. + want := map[string]int{"2.1": 1, "2.0": 1, "1.1": 1} + if got := u.pomRequests(); len(got) != len(want) || got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 { + t.Errorf("pom requests = %v, want %v", got, want) + } + + // Times are stored, so the next request makes no .pom requests. + ver, err := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if err != nil || ver == nil || !ver.PublishedAt.Valid { + t.Fatalf("time of 2.1 not stored: %v %v", ver, err) + } + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != body { + t.Errorf("second response differs:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests after second metadata request = %d, want 3", n) + } +} + +func TestMavenCooldownRecomputesLatest(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"1.0", "1.1", "2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if !strings.Contains(body, "1.1") || !strings.Contains(body, "1.1") { + t.Errorf("latest and release should be 1.1:\n%s", body) + } +} + +func TestMavenCooldownDropsLatestWhenNothingIsLeft(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.versions = []string{"2.0", "2.1"} + u.latest = "2.1" + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String() + if strings.Contains(body, "") || strings.Contains(body, "") || strings.Contains(body, "") { + t.Errorf("no version is out of cooldown:\n%s", body) + } + if !strings.Contains(body, "\n ") { + t.Errorf("version list not emptied in place:\n%s", body) + } +} + +func TestMavenCooldownChecksumsMatchMetadata(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + body := getMaven(h, mavenTestDir+"maven-metadata.xml").Body.Bytes() + for suffix, newHash := range map[string]func() hash.Hash{ + ".md5": md5.New, ".sha1": sha1.New, ".sha256": sha256.New, ".sha512": sha512.New, + } { + sum := newHash() + _, _ = sum.Write(body) + w := getMaven(h, mavenTestDir+"maven-metadata.xml"+suffix) + if w.Code != http.StatusOK || w.Body.String() != hex.EncodeToString(sum.Sum(nil)) { + t.Errorf("%s: status = %d body %q, want checksum of served metadata", suffix, w.Code, w.Body) + } + } + // A signature of the upstream metadata would not match. + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusNotFound { + t.Errorf(".asc status = %d, want 404", w.Code) + } + + // Unchanged metadata keeps its signature. + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + if w := getMaven(h, mavenTestDir+"maven-metadata.xml.asc"); w.Code != http.StatusOK || w.Body.String() != "signature" { + t.Errorf("unchanged .asc: status = %d body %q, want upstream's", w.Code, w.Body) + } +} + +func TestMavenCooldownRefusesFiles(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + for _, file := range []string{"2.1/lib-2.1.pom", "2.1/lib-2.1.jar", "2.1/lib-2.1.jar.sha1", "2.0/lib-2.0-sources.jar"} { + w := getMaven(h, mavenTestDir+file) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), errMavenCooldown) { + t.Errorf("%s: status = %d body %q, want 403 cooldown", file, w.Code, w.Body) + } + } + for _, file := range []string{"1.1/lib-1.1.jar", "3.0-SNAPSHOT/lib-3.0-SNAPSHOT.jar"} { + if w := getMaven(h, mavenTestDir+file); w.Code != http.StatusOK { + t.Errorf("%s: status = %d, want 200", file, w.Code) + } + } + // One .pom request per version, none for the SNAPSHOT. + if got := u.pomRequests(); got["2.1"] != 1 || got["2.0"] != 1 || got["1.1"] != 1 || got["3.0-SNAPSHOT"] != 0 { + t.Errorf("pom requests = %v", got) + } +} + +func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) + // The times would filter 2.0 and 2.1 if they were read. + h, _ := newMavenCooldownHandler(t, u) + + w := getMaven(h, mavenTestDir+"maven-metadata.xml") + if w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.1"] = http.StatusNotFound + h, proxy := newMavenCooldownHandler(t, u) + + // A missing .pom gives no time: 2.1 stays listed, and the walk goes on + // to 2.0 rather than stopping at the error page's old date. + versions := mavenMetadataVersions(t, getMaven(h, mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 2.1 3.0-SNAPSHOT" { + t.Errorf("versions = %s, want 1.0 1.1 2.1 3.0-SNAPSHOT", got) + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 404 stored: %v", ver.PublishedAt.Time) + } + + // Any other error fails closed. + u.pomStatus["2.1"] = http.StatusTooManyRequests + for _, p := range []string{"maven-metadata.xml", "maven-metadata.xml.sha1", "2.1/lib-2.1.jar"} { + if w := getMaven(h, mavenTestDir+p); w.Code != http.StatusBadGateway { + t.Errorf("%s: status = %d, want 502", p, w.Code) + } + } + if ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")); ver != nil && ver.PublishedAt.Valid { + t.Errorf("time of a 429 stored: %v", ver.PublishedAt.Time) + } +} + +func TestMavenCooldownDisabledPassesThrough(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = nil + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownPackageOverride(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + proxy.Cooldown = &cooldown.Config{Default: "3d", Packages: map[string]string{"pkg:maven/com.example/lib": "0"}} + + if w := getMaven(h, mavenTestDir+"maven-metadata.xml"); w.Body.String() != u.metadata() { + t.Errorf("metadata changed:\n%s", w.Body) + } + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusOK { + t.Errorf("jar status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +func TestMavenCooldownNamedRepositoryStoresItsOwnTimes(t *testing.T) { + u := newMavenCooldownUpstream(t) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.server.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + // The named repository is the same server under /portal, which serves + // the same files. + h := NewMavenHandler(proxy, "http://localhost", u.server.URL, u.server.URL+"/portal"). + WithRepositories(map[string]string{"mirror": u.server.URL + "/portal"}).Routes() + + for range 2 { + versions := mavenMetadataVersions(t, getMaven(h, "/mirror"+mavenTestDir+"maven-metadata.xml").Body.String()) + if got := strings.Join(versions, " "); got != "1.0 1.1 3.0-SNAPSHOT" { + t.Fatalf("versions = %s, want 1.0 1.1 3.0-SNAPSHOT", got) + } + } + if w := getMaven(h, "/mirror"+mavenTestDir+"2.1/lib-2.1.jar"); w.Code != http.StatusForbidden { + t.Errorf("jar status = %d, want 403", w.Code) + } + if n := u.totalPOMRequests(); n != 3 { + t.Errorf("pom requests = %d, want 3", n) + } + + // The default repository doesn't use the named one's times. + _ = getMaven(h, mavenTestDir+"maven-metadata.xml") + if n := u.totalPOMRequests(); n != 6 { + t.Errorf("pom requests after default metadata = %d, want 6", n) + } +} + +func TestMavenMetadataWithoutCompact(t *testing.T) { + in := `2212` + doc, err := parseMavenMetadata([]byte(in)) + if err != nil { + t.Fatal(err) + } + got := string(doc.without([]byte(in), map[string]bool{"2": true})) + want := `111` + if got != want { + t.Errorf("got %s\nwant %s", got, want) + } +} From e6c4a3e57ae3ffc1eb2d13f0edfd2e12f5626b6a Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:38:28 +0200 Subject: [PATCH 3/6] Document Maven cooldown Includes the caveats from #132: Nexus and Artifactory dates, Google's Maven Central mirror, version-ordered lists and SNAPSHOTs. --- README.md | 4 +++- docs/configuration.md | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3421dffb..aac9ff57 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Hex | Elixir | Yes* | ✓ | | pub.dev | Dart | Yes | ✓ | | PyPI | Python | Yes | ✓ | -| Maven | Java | | ✓ | +| Maven | Java | Yes‡ | ✓ | | Gradle Build Cache | Java/Kotlin | | ✓ | | NuGet | .NET | Yes | ✓ | | Composer | PHP | Yes | ✓ | @@ -79,6 +79,8 @@ Cooldown requires publish timestamps in metadata. Registries without a "Yes" in † Go cooldown uses the time in each version's `.info`, which is the commit time of the tagged revision, not when the tag was pushed. +‡ Maven cooldown uses the `Last-Modified` header of each version's `.pom`. On Maven Central that is the deploy time; a Nexus or Artifactory upstream may report when it cached the file instead. SNAPSHOT versions are not held back. + ## Install ```bash diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec69..d9322924 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -980,10 +980,19 @@ Exact `packages` entries take precedence over patterns. When several patterns ma Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. -Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex and Go. These ecosystems include publish timestamps in their metadata. +Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, Hex, Go and Maven. Most of these ecosystems include publish timestamps in their metadata; Go and Maven are described below. Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version with a `HEAD` request and stores the time, for each repository under `upstream.maven_repositories` separately. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through; a version without a `.pom` is let through. + +Maven caveats: + +- A Nexus or Artifactory upstream may report when it cached the `.pom` rather than when it was published, so a version can stay in cooldown longer than it should. +- On Google's Maven Central mirror, `Last-Modified` is when the file reached the mirror's bucket, a few hours after it was published, so versions stay in cooldown a little longer. The upload time is in `x-goog-meta-last-modified`, which the proxy does not read. It is missing for files from the mirror's 2019 bulk import. +- Maven Central lists versions in version order, not in the order they were published. A patch to an older release line published after a newer release that is out of cooldown stays listed while it is in cooldown, but its files are still refused. +- SNAPSHOT versions are not held back. + With cooldown on, the proxy asks npm for full packuments, since the abbreviated ones it uses otherwise carry no publish times. Abbreviated packuments cached before cooldown was turned on are fetched again in full rather than served, including as a stale copy when the upstream is down. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. From 78f614e82cf7b03222f248e5ec05f9e13611044f Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 19:50:34 +0200 Subject: [PATCH 4/6] Read a Maven version's time from its .pom download A build fetching pinned versions never reads maven-metadata.xml, so each version's time was read with a HEAD before its download, roughly doubling the requests to Maven Central, which answered a fresh build with 429 and a 30-minute Retry-After. The .pom's own response now gives the time; Maven fetches the .pom before a version's other files, so their check reads the stored time. A HEAD is only sent for a .pom served from the cache or a file requested before its .pom. A 429 on that HEAD now answers 503 with the upstream's Retry-After, as other held upstream requests do, instead of 502. --- internal/handler/handler.go | 41 ++++++++++++++- internal/handler/maven.go | 14 ++++-- internal/handler/maven_cooldown.go | 34 +++++++++++++ internal/handler/maven_cooldown_test.go | 66 ++++++++++++++++++++++++- 4 files changed, 148 insertions(+), 7 deletions(-) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8bcac24f..b5c34ef6 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1965,7 +1965,7 @@ func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, eco if p.StreamArtifacts { return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, declared, func(fetchCtx context.Context) (*fetch.Artifact, error) { - return p.Fetcher.FetchWithHeaders(fetchCtx, downloadURL, headers) + return p.fetchUpstream(fetchCtx, downloadURL, headers) }) } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { @@ -2040,7 +2040,7 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) fetchStart := time.Now() - artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + artifact, err := p.fetchUpstream(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed", metrics.FailureDetail{ @@ -2054,6 +2054,43 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, declared, artifact) } +type upstreamLastModifiedKey struct{} + +// WithUpstreamLastModified returns a context under which an artifact fetched +// from upstream reports its Last-Modified to fn, so a handler can learn a +// version's publish time from the download itself rather than with a +// request of its own. fn is not called for a cache hit, or when the +// response has no usable Last-Modified. +func WithUpstreamLastModified(ctx context.Context, fn func(time.Time)) context.Context { + return context.WithValue(ctx, upstreamLastModifiedKey{}, fn) +} + +// observedFetcher is the part of fetch.Fetcher and its circuit breaker that +// reports response headers. +type observedFetcher interface { + FetchObservedWithHeaders(ctx context.Context, url string, headers http.Header) (*fetch.ObservedArtifact, error) +} + +// fetchUpstream fetches downloadURL, reporting its Last-Modified when ctx +// asks for it. +func (p *Proxy) fetchUpstream(ctx context.Context, downloadURL string, headers http.Header) (*fetch.Artifact, error) { + report, _ := ctx.Value(upstreamLastModifiedKey{}).(func(time.Time)) + observer, ok := p.Fetcher.(observedFetcher) + if report == nil || !ok { + return p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) + } + observed, err := observer.FetchObservedWithHeaders(ctx, downloadURL, headers) + if err != nil { + return nil, err + } + if observed.Observation != nil { + if t, err := http.ParseTime(observed.Observation.Headers.Get(headerLastModified)); err == nil { + report(t) + } + } + return observed.Artifact, nil +} + // streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its // body to the caller without storing it. // diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 48859065..c201e74b 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -225,7 +225,11 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so return } - if h.checkCooldown(w, r, source, urlPath, version) { + // The .pom of a version whose time isn't known yet carries it in its + // own Last-Modified, so it is checked once fetched, without a request + // of its own. Maven fetches the .pom before a version's other files. + ctx, checkAfter := h.cooldownFromDownload(r.Context(), source, urlPath, version) + if !checkAfter && h.checkCooldown(w, r, source, urlPath, version) { return } @@ -238,19 +242,23 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, so upstreamURL := fmt.Sprintf("%s/%s", source.upstreamURL, urlPath) cacheFilename := source.cacheFilename(filename) - result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, upstreamURL) + result, err := h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, upstreamURL) if err != nil && source.fallbackURL != "" { if errors.Is(err, ErrUpstreamNotFound) { pluginPortalURL := fmt.Sprintf("%s/%s", source.fallbackURL, urlPath) h.proxy.Logger.Info("maven artifact not found in primary upstream, trying Gradle Plugin Portal", "group", group, "artifact", artifact, "version", version, "filename", filename) - result, err = h.proxy.GetOrFetchArtifactFromURL(r.Context(), "maven", name, version, cacheFilename, pluginPortalURL) + result, err = h.proxy.GetOrFetchArtifactFromURL(ctx, "maven", name, version, cacheFilename, pluginPortalURL) } } if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch artifact") return } + if checkAfter && h.checkCooldown(w, r, source, urlPath, version) { + _ = result.Reader.Close() + return + } ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index ca5a03a0..d3388c7b 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -129,6 +129,34 @@ func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, arti return published, nil } +// cooldownFromDownload returns a context that stores the publish time of a +// version from the download of its .pom, and whether the cooldown check is +// to run after the download. It does so only for the .pom of a version +// whose time is not stored yet; for anything else the check runs before. +func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { + if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { + return ctx, false + } + artifactDir := path.Dir(path.Dir(urlPath)) + group, artifact, ok := mavenCoordinates(artifactDir) + if !ok || !usableMavenVersion(version) || path.Base(urlPath) != artifact+"-"+version+".pom" { + return ctx, false + } + versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) + if !store { + return ctx, false + } + if ver, err := h.proxy.DB.GetVersionByPURL(versionPURL); err == nil && ver != nil && ver.PublishedAt.Valid { + return ctx, false + } + return WithUpstreamLastModified(ctx, func(published time.Time) { + if err := h.proxy.DB.SetVersionPublishedAt(versionPURL, packagePURL, published); err != nil { + h.proxy.Logger.Warn("cooldown: could not store maven publish time", + "group", group, "artifact", artifact, "version", version, "error", err) + } + }), true +} + // pomLastModified sends a HEAD request for a .pom and returns its // Last-Modified. found is false when upstream doesn't have the file. Only a // 2xx response's header is used: an error page's date says nothing about the @@ -146,6 +174,12 @@ func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (publ switch { case resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone: return time.Time{}, false, nil + case resp.StatusCode == http.StatusTooManyRequests && h.proxy.RateLimits != nil: + // The gate has just noted the upstream's Retry-After; pass it on. + if err := h.proxy.RateLimits.Held(pomURL); err != nil { + return time.Time{}, false, err + } + return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) case resp.StatusCode < 200 || resp.StatusCode > 299: return time.Time{}, false, fmt.Errorf("reading pom time: upstream returned %d", resp.StatusCode) } diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go index 74c42683..9a8e2719 100644 --- a/internal/handler/maven_cooldown_test.go +++ b/internal/handler/maven_cooldown_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/git-pkgs/cooldown" + registryclient "github.com/git-pkgs/registries/client" "github.com/git-pkgs/registries/fetch" ) @@ -32,8 +33,9 @@ type mavenCooldownUpstream struct { times map[string]time.Time pomStatus map[string]int - mu sync.Mutex - poms map[string]int + mu sync.Mutex + poms map[string]int + heads int } func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { @@ -84,11 +86,17 @@ func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(file, ".pom") { u.mu.Lock() u.poms[version]++ + if r.Method == http.MethodHead { + u.heads++ + } u.mu.Unlock() w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) if status := u.pomStatus[version]; status != 0 { // An error page dated long ago must not count as an old version. w.Header().Set(headerLastModified, time.Unix(0, 0).UTC().Format(http.TimeFormat)) + if status == http.StatusTooManyRequests { + w.Header().Set("Retry-After", "60") + } w.WriteHeader(status) return } @@ -265,6 +273,42 @@ func TestMavenCooldownRefusesFiles(t *testing.T) { } } +// TestMavenCooldownPOMDownloadCarriesItsTime checks that a build fetching +// pinned versions, which never reads maven-metadata.xml, costs no requests +// beyond its downloads: the .pom's own Last-Modified gives the version's +// time, and Maven fetches the .pom before the version's other files. +func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, proxy := newMavenCooldownHandler(t, u) + + for _, c := range []struct { + file string + want int + }{ + {"2.1/lib-2.1.pom", http.StatusForbidden}, + {"2.1/lib-2.1.jar", http.StatusForbidden}, + {"1.0/lib-1.0.pom", http.StatusOK}, + {"1.0/lib-1.0.jar", http.StatusOK}, + } { + if w := getMaven(h, mavenTestDir+c.file); w.Code != c.want { + t.Errorf("%s: status = %d, want %d", c.file, w.Code, c.want) + } + } + u.mu.Lock() + heads := u.heads + u.mu.Unlock() + if heads != 0 { + t.Errorf("HEAD requests = %d, want 0", heads) + } + if got := u.pomRequests(); got["2.1"] != 1 || got["1.0"] != 1 { + t.Errorf("pom requests = %v, want one GET each", got) + } + ver, _ := proxy.DB.GetVersionByPURL(canonicalVersionPURL("maven", "com.example:lib", "2.1")) + if ver == nil || !ver.PublishedAt.Valid || !ver.PublishedAt.Time.Equal(u.times["2.1"].Truncate(time.Second)) { + t.Errorf("stored time of 2.1 = %v, want %v", ver, u.times["2.1"]) + } +} + func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { u := newMavenCooldownUpstream(t) u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) @@ -307,6 +351,24 @@ func TestMavenCooldownIgnoresNon2xxPOMResponses(t *testing.T) { } } +// TestMavenCooldownRateLimitedPOMIsHeld checks that a 429 on a .pom time +// request is answered like any held upstream request, with 503 and the +// upstream's Retry-After, so clients wait instead of failing the build. +func TestMavenCooldownRateLimitedPOMIsHeld(t *testing.T) { + u := newMavenCooldownUpstream(t) + u.pomStatus["2.0"] = http.StatusTooManyRequests + h, proxy := newMavenCooldownHandler(t, u) + gate := registryclient.NewRateLimitGate(u.server.Client().Transport) + proxy.HTTPClient = &http.Client{Transport: gate} + proxy.RateLimits = gate + + // Not the .pom, so its time is read with a HEAD. + w := getMaven(h, mavenTestDir+"2.0/lib-2.0.jar") + if w.Code != http.StatusServiceUnavailable || w.Header().Get("Retry-After") == "" { + t.Errorf("status = %d, Retry-After %q; want 503 with Retry-After", w.Code, w.Header().Get("Retry-After")) + } +} + func TestMavenCooldownDisabledPassesThrough(t *testing.T) { u := newMavenCooldownUpstream(t) h, proxy := newMavenCooldownHandler(t, u) From f81274b499234d5149c490b894f7ceb1d77ceb9e Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 20:07:53 +0200 Subject: [PATCH 5/6] Describe how Maven publish times are read --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index d9322924..d052ff91 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -984,7 +984,7 @@ Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyG Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version with a `HEAD` request and stores the time, for each repository under `upstream.maven_repositories` separately. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` itself is downloaded, which Maven does before a version's other files, and otherwise with a `HEAD` request. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: From faecb56c851709d0fcdfcdb38a7f818305a09f22 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Fri, 23 Oct 2026 20:53:06 +0200 Subject: [PATCH 6/6] Polish Maven cooldown after review - Name the proxy in the User-Agent of publish-time HEAD requests, as its artifact downloads do, rather than Go's default, which Maven Central asks tools not to send. - Take a version's time from its Gradle .module as well as its .pom, so either one fetched first carries it. - Serve filtered metadata through the shared rewrite path, with gzip and the stale warning like the other ecosystems. - Say in the docs that a refused .pom or .module is still cached. --- docs/configuration.md | 2 +- internal/handler/maven_cooldown.go | 38 +++++------ internal/handler/maven_cooldown_test.go | 91 ++++++++++++++++++++++++- internal/handler/rewrite_cache.go | 8 ++- 4 files changed, 114 insertions(+), 25 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d052ff91..226d23e4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -984,7 +984,7 @@ Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyG Go has no publish time. The module proxy's `.info` gives the commit time of the revision a version points at, so a version tagged on an old commit counts as old. The proxy reads each listed version's `.info` once, stores the time, and leaves versions in cooldown out of `@v/list` and `@latest`. When upstream's `@latest` is in cooldown, it serves the newest release out of cooldown instead. Requests for a version in cooldown, including its `.info`, `.mod` and `.zip`, and a branch or commit query that resolves to one, get 403 rather than 404, because the go command moves on to the next `GOPROXY` entry, such as `direct`, after a 404. If a version's `.info` can't be fetched, the list or download fails with 502 instead of letting the version through. -Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` itself is downloaded, which Maven does before a version's other files, and otherwise with a `HEAD` request. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. +Maven metadata has no publish time per version. The proxy uses the `Last-Modified` header of each version's `.pom`, which on Maven Central is the time the version was deployed. It reads the header once per version and stores the time, for each repository under `upstream.maven_repositories` separately: from the response when the `.pom` or Gradle `.module` itself is downloaded, which clients do before a version's other files, and otherwise with a `HEAD` request. A `.pom` or `.module` refused this way is still cached, so it counts toward `max_size` before the version leaves cooldown. To keep requests to the upstream down, it only checks versions that can still be in cooldown: none when the metadata's `` is older than the cooldown, and otherwise the versions from the end of the list back to the first one that is old enough. Versions in cooldown are left out of `maven-metadata.xml`, `` and `` are set to the last versions left, and the metadata's `.md5`, `.sha1`, `.sha256` and `.sha512` files are computed from what is served. Requests for any file of a version in cooldown get 403, as for a version on the denylist. If a `.pom` can't be checked, the metadata or download fails with 502 instead of letting the version through, or with 503 and `Retry-After` when the upstream rate-limits the proxy; a version without a `.pom` is let through. Maven caveats: diff --git a/internal/handler/maven_cooldown.go b/internal/handler/maven_cooldown.go index d3388c7b..d67d9c9c 100644 --- a/internal/handler/maven_cooldown.go +++ b/internal/handler/maven_cooldown.go @@ -37,6 +37,10 @@ import ( const errMavenCooldown = "version is in cooldown" +// mavenTimeUserAgent is the User-Agent of publish-time requests, the one +// the artifact fetcher sends. +const mavenTimeUserAgent = "git-pkgs-proxy/1.0" + const ( mavenMetadataFile = "maven-metadata.xml" mavenSnapshot = "-SNAPSHOT" @@ -130,16 +134,19 @@ func (h *MavenHandler) publishTime(ctx context.Context, source mavenSource, arti } // cooldownFromDownload returns a context that stores the publish time of a -// version from the download of its .pom, and whether the cooldown check is -// to run after the download. It does so only for the .pom of a version -// whose time is not stored yet; for anything else the check runs before. +// version from the download of its .pom or Gradle .module, and whether the +// cooldown check is to run after the download. It does so only for those +// files of a version whose time is not stored yet; for anything else the +// check runs before. Both files are deployed with the version, and +// whichever a client fetches first gives its time. func (h *MavenHandler) cooldownFromDownload(ctx context.Context, source mavenSource, urlPath, version string) (context.Context, bool) { if !h.cooldownEnabled() || strings.HasSuffix(version, mavenSnapshot) { return ctx, false } artifactDir := path.Dir(path.Dir(urlPath)) group, artifact, ok := mavenCoordinates(artifactDir) - if !ok || !usableMavenVersion(version) || path.Base(urlPath) != artifact+"-"+version+".pom" { + if base := artifact + "-" + version; !ok || !usableMavenVersion(version) || + (path.Base(urlPath) != base+".pom" && path.Base(urlPath) != base+".module") { return ctx, false } versionPURL, packagePURL, store := h.mavenTimeKey(source, group, artifact, version) @@ -166,6 +173,9 @@ func (h *MavenHandler) pomLastModified(ctx context.Context, pomURL string) (publ if err != nil { return time.Time{}, false, err } + // Name the proxy as its artifact downloads do; Maven Central asks + // tools not to send an HTTP library's default. + req.Header.Set("User-Agent", mavenTimeUserAgent) resp, err := h.proxy.HTTPClient.Do(req) if err != nil { return time.Time{}, false, fmt.Errorf("reading pom time: %w", err) @@ -258,10 +268,13 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ switch { case suffix == "": + // The metadata cache records JSON when an upstream sent no type, + // and maven-metadata.xml is always XML. if contentType == "" || contentType == contentTypeJSON { contentType = "text/xml" } - serveMavenMetadata(w, r, m, contentType) + markStale(w, h.proxy.lookupCachedMeta("maven", cacheKey).stale) + serveRewrittenMetadataAs(w, r, m, contentType) case mavenChecksums[suffix] != nil: sum := mavenChecksums[suffix]() _, _ = sum.Write(m.body) @@ -276,21 +289,6 @@ func (h *MavenHandler) serveCooldownMetadata(w http.ResponseWriter, r *http.Requ return true } -// serveMavenMetadata writes filtered metadata with its validators, or 304 -// when they show the client's copy is current. -func serveMavenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { - w.Header().Set(headerETag, m.etag) - if !m.lastModified.IsZero() { - w.Header().Set(headerLastModified, m.lastModified.UTC().Format(http.TimeFormat)) - } - if notModified(r, m.etag, m.lastModified) { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set(headerContentType, contentType) - _, _ = w.Write(m.body) -} - // filterMetadata removes versions in cooldown from an artifact's // maven-metadata.xml and points and at the newest // versions left. Metadata without a version list, such as a group's plugin diff --git a/internal/handler/maven_cooldown_test.go b/internal/handler/maven_cooldown_test.go index 9a8e2719..828d1ffa 100644 --- a/internal/handler/maven_cooldown_test.go +++ b/internal/handler/maven_cooldown_test.go @@ -1,14 +1,18 @@ package handler import ( + "compress/gzip" "crypto/md5" "crypto/sha1" "crypto/sha256" "crypto/sha512" "encoding/hex" + "fmt" "hash" + "io" "net/http" "net/http/httptest" + "slices" "strings" "sync" "testing" @@ -33,9 +37,10 @@ type mavenCooldownUpstream struct { times map[string]time.Time pomStatus map[string]int - mu sync.Mutex - poms map[string]int - heads int + mu sync.Mutex + poms map[string]int + heads int + userAgents []string } func newMavenCooldownUpstream(t *testing.T) *mavenCooldownUpstream { @@ -83,11 +88,15 @@ func (u *mavenCooldownUpstream) serve(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } + if strings.HasSuffix(file, ".module") { + w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) + } if strings.HasSuffix(file, ".pom") { u.mu.Lock() u.poms[version]++ if r.Method == http.MethodHead { u.heads++ + u.userAgents = append(u.userAgents, r.UserAgent()) } u.mu.Unlock() w.Header().Set(headerLastModified, published.UTC().Format(http.TimeFormat)) @@ -309,6 +318,82 @@ func TestMavenCooldownPOMDownloadCarriesItsTime(t *testing.T) { } } +// TestMavenCooldownModuleDownloadCarriesItsTime checks that a Gradle +// .module fetched before the .pom gives the version's time as the .pom +// would. +func TestMavenCooldownModuleDownloadCarriesItsTime(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + if w := getMaven(h, mavenTestDir+"2.1/lib-2.1.module"); w.Code != http.StatusForbidden { + t.Errorf("2.1 .module: status = %d, want 403", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.module"); w.Code != http.StatusOK { + t.Errorf("1.0 .module: status = %d, want 200", w.Code) + } + if w := getMaven(h, mavenTestDir+"1.0/lib-1.0.jar"); w.Code != http.StatusOK { + t.Errorf("1.0 .jar: status = %d, want 200", w.Code) + } + if n := u.totalPOMRequests(); n != 0 { + t.Errorf("pom requests = %d, want 0", n) + } +} + +// TestMavenCooldownTimeRequestsNameTheProxy checks the User-Agent of the +// HEAD requests for publish times: Maven Central asks tools to name +// themselves rather than send an HTTP library's default. +func TestMavenCooldownTimeRequestsNameTheProxy(t *testing.T) { + u := newMavenCooldownUpstream(t) + h, _ := newMavenCooldownHandler(t, u) + + getMaven(h, mavenTestDir+"maven-metadata.xml") + u.mu.Lock() + defer u.mu.Unlock() + if len(u.userAgents) == 0 { + t.Fatal("no HEAD requests") + } + for _, ua := range u.userAgents { + if ua != mavenTimeUserAgent { + t.Errorf("User-Agent = %q, want %q", ua, mavenTimeUserAgent) + } + } +} + +// TestMavenCooldownMetadataServedLikeOtherRewrites checks that filtered +// metadata is gzip-compressed for clients that accept it, as the other +// ecosystems' rewritten metadata is. +func TestMavenCooldownMetadataServedLikeOtherRewrites(t *testing.T) { + u := newMavenCooldownUpstream(t) + // Enough versions that the document compresses. + for i := range 50 { + v := fmt.Sprintf("0.%d", i) + u.versions = append([]string{v}, u.versions...) + u.times[v] = time.Now().Add(-100 * 24 * time.Hour) + } + h, proxy := newMavenCooldownHandler(t, u) + // The server always sets up the rewrite cache, which keeps the gzip copy. + proxy.SetMetadataRewriteCacheSize(1 << 20) + + req := httptest.NewRequest(http.MethodGet, mavenTestDir+"maven-metadata.xml", nil) + req.Header.Set(headerAcceptEncoding, "gzip") + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "gzip" { + t.Fatalf("status = %d, Content-Encoding %q; want gzip", w.Code, w.Header().Get(headerContentEncoding)) + } + if ct := w.Header().Get(headerContentType); !strings.Contains(ct, "xml") { + t.Errorf("Content-Type = %q, want XML", ct) + } + zr, err := gzip.NewReader(w.Body) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(zr) + if versions := mavenMetadataVersions(t, string(body)); slices.Contains(versions, "2.1") { + t.Errorf("2.1 listed in the gzip body: %v", versions) + } +} + func TestMavenCooldownOldLastUpdatedSkipsPOMs(t *testing.T) { u := newMavenCooldownUpstream(t) u.lastUpdated = time.Now().Add(-4 * 24 * time.Hour) diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go index 0569a9cd..991443af 100644 --- a/internal/handler/rewrite_cache.go +++ b/internal/handler/rewrite_cache.go @@ -402,6 +402,12 @@ func (p *Proxy) storedRewrite(ecosystem, cacheKey, proxyURL, name string) (rewri // accepting gzip gets the compressed copy when there is one. Being another // representation, it has an ETag of its own. func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenMetadata) { + serveRewrittenMetadataAs(w, r, m, contentTypeJSON) +} + +// serveRewrittenMetadataAs is serveRewrittenMetadata for a document of +// another content type. +func serveRewrittenMetadataAs(w http.ResponseWriter, r *http.Request, m rewrittenMetadata, contentType string) { body, etag := m.body, m.etag compressed := m.gzipped != nil && acceptsGzip(r) if compressed { @@ -416,7 +422,7 @@ func serveRewrittenMetadata(w http.ResponseWriter, r *http.Request, m rewrittenM w.WriteHeader(http.StatusNotModified) return } - w.Header().Set(headerContentType, contentTypeJSON) + w.Header().Set(headerContentType, contentType) if compressed { w.Header().Set(headerContentEncoding, "gzip") }