From fdd0c814eb69403a5fe2a566ac15945d95836aab Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:39:23 +0100 Subject: [PATCH 1/2] Build in version-pinned paths for well-known download hosts Generic upstreams pointed at archive.apache.org, nodejs.org or dl.google.com/go now cache their versioned downloads as artifacts without any generic_artifacts patterns. Configured patterns add to the built-in ones. --- README.md | 18 ++++++++---- docs/configuration.md | 17 ++++++++--- internal/handler/generic.go | 45 ++++++++++++++++++++++------- internal/handler/generic_test.go | 49 ++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 020916cc..5b7d9d66 100644 --- a/README.md +++ b/README.md @@ -702,24 +702,30 @@ Commit a `mise.lock` and install with `mise install --locked` so pinned installs need no API call at all. Add a bearer token for `https://api.github.com` under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. -Tools mise downloads from elsewhere, such as Maven from `archive.apache.org`, -can be cached the same way by declaring their version-pinned paths in -`upstream.generic_artifacts` (see [configuration](docs/configuration.md)): +Maven, mvnd, Node and Go downloads are cached the same way once their hosts +are configured as generic upstreams; their version-pinned paths are built in +(see [configuration](docs/configuration.md)): ```yaml upstream: generic: apache: "https://archive.apache.org" - generic_artifacts: - apache: - - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + nodejs: "https://nodejs.org" + golang: "https://dl.google.com/go" ``` ```toml +[settings] +node.mirror_url = "http://localhost:8080/generic/nodejs/dist/" +go.download_mirror = "http://localhost:8080/generic/golang" + [settings.url_replacements] "https://archive.apache.org/" = "http://localhost:8080/generic/apache/" ``` +Other download hosts need their version-pinned paths declared in +`upstream.generic_artifacts`. + ## Configuration The proxy can be configured via: diff --git a/docs/configuration.md b/docs/configuration.md index 41ba4d0d..ce8aa30a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -361,8 +361,17 @@ version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is down. +Upstreams pointed at these hosts get version-pinned paths built in, whatever +the upstream is named: + +| Upstream URL | Cached as artifacts | Package name | +|---|---|---| +| `https://archive.apache.org` | `dist/{project dirs}/{version}/[binaries/]{file}`, where `{version}` starts with a digit | the project dirs, e.g. `maven/maven-3`, `maven/mvnd` | +| `https://nodejs.org` | `dist/v{version}/{file}` | the upstream name | +| `https://dl.google.com/go` | `go{version}.{os}-{arch}.tar.gz`, `.zip` and their `.sha256` | the upstream name | + Other upstreams can declare their own version-pinned paths in -`generic_artifacts`. Each pattern is a regular expression that must match the +`generic_artifacts`, which add to any built-in ones. Each pattern is a regular expression that must match the whole path after `/generic/{name}/`, with `version` and `file` named groups (single path segments) and an optional `name` group for the package name, which otherwise is the upstream name. Matching downloads are cached like @@ -371,10 +380,10 @@ release assets and listed under the `generic` ecosystem: ```yaml upstream: generic: - apache: "https://archive.apache.org" + eclipse: "https://download.eclipse.org" generic_artifacts: - apache: - - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + eclipse: + - 'jdtls/milestones/(?P[^/]+)/(?P[^/]+)' ``` Only list paths whose content never changes for a given version: a match is diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 67ae94ad..954e0911 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -28,6 +28,23 @@ const ( // and served without revalidation once fetched. var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/download/([^/]+)/([^/]+)$`) +// builtinGenericArtifacts maps well-known download hosts to patterns for +// their version-pinned paths. They apply to any generic upstream configured +// with that base URL, before the upstream's own generic_artifacts. +var builtinGenericArtifacts = map[string][]string{ + // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. + "https://archive.apache.org": { + `dist/(?P.+?)/(?P[0-9][^/]*)/(?:binaries/)?(?P[^/]+)`, + }, + "https://nodejs.org": { + `dist/(?Pv[0-9][^/]*)/(?P[^/]+)`, + }, + // The version is only in the filename: go1.23.2.linux-amd64.tar.gz. + "https://dl.google.com/go": { + `(?Pgo(?P[0-9][^/]*?)\.[a-z0-9]+-[a-z0-9]+\.(?:tar\.gz|zip)(?:\.sha256)?)`, + }, +} + // GenericHandler proxies plain HTTP downloads from configured upstream base // URLs. Each configured upstream is mounted at /generic/{name}/ and the // remaining request path (and query string) is appended to the upstream URL. @@ -38,7 +55,7 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // assets, and is pointed at by URL-rewriting settings on the client. // // Release-asset paths ({owner}/{repo}/releases/download/{tag}/{asset}), and -// paths matching the upstream's configured artifact patterns, are +// paths matching the upstream's built-in or configured artifact patterns, are // version-pinned and cached in the shared artifact cache, so they keep being // served when the upstream is unreachable. Every other path is served through // the metadata cache: fresh within the metadata TTL, revalidated with the @@ -61,24 +78,30 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { - h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") + upstreamURL = strings.TrimSuffix(upstreamURL, "/") + h.repositories[name] = upstreamURL + h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) } if len(artifactPatterns) > 0 { for name, patterns := range artifactPatterns[0] { - for _, pattern := range patterns { - re, err := config.CompileGenericArtifactPattern(pattern) - if err != nil { - proxy.Logger.Warn("skipping generic artifact pattern", - "repository", name, "pattern", pattern, "error", err) - continue - } - h.artifactPatterns[name] = append(h.artifactPatterns[name], re) - } + h.addArtifactPatterns(name, patterns) } } return h } +func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { + for _, pattern := range patterns { + re, err := config.CompileGenericArtifactPattern(pattern) + if err != nil { + h.proxy.Logger.Warn("skipping generic artifact pattern", + "repository", name, "pattern", pattern, "error", err) + continue + } + h.artifactPatterns[name] = append(h.artifactPatterns[name], re) + } +} + // Routes returns the HTTP handler for generic download requests. // Mount this at /generic on your router. func (h *GenericHandler) Routes() http.Handler { diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 3e45fb94..1292e8c2 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -105,6 +105,55 @@ func TestMatchArtifactPattern(t *testing.T) { } } +func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { + h := NewGenericHandler(testProxy(), map[string]string{ + "apache": "https://archive.apache.org/", + "node": "https://nodejs.org", + "golang": "https://dl.google.com/go", + "mirror": "https://mirror.example.com", + }, map[string][]string{ + "apache": {`extra/(?P[^/]+)/(?P[^/]+)`}, + }) + + tests := []struct { + repository string + path string + want genericArtifact + ok bool + }{ + {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", + genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, + {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz.sha512", + genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + {"apache", "dist/maven/mvnd/1.0.6/maven-mvnd-1.0.6-linux-amd64.tar.gz", + genericArtifact{name: "maven/mvnd", version: "1.0.6", filename: "maven-mvnd-1.0.6-linux-amd64.tar.gz"}, true}, + {"apache", "dist/maven/maven-3/", genericArtifact{}, false}, + {"apache", "dist/maven/maven-3/3.9.9/source/apache-maven-3.9.9-src.tar.gz", genericArtifact{}, false}, + {"apache", "extra/1.0/file.txt", genericArtifact{name: "apache", version: "1.0", filename: "file.txt"}, true}, + {"node", "dist/v22.11.0/node-v22.11.0-linux-x64.tar.gz", + genericArtifact{name: "node", version: "v22.11.0", filename: "node-v22.11.0-linux-x64.tar.gz"}, true}, + {"node", "dist/v22.11.0/SHASUMS256.txt", + genericArtifact{name: "node", version: "v22.11.0", filename: "SHASUMS256.txt"}, true}, + {"node", "dist/index.json", genericArtifact{}, false}, + {"golang", "go1.23.2.linux-amd64.tar.gz", + genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.linux-amd64.tar.gz"}, true}, + {"golang", "go1.23.2.darwin-arm64.tar.gz.sha256", + genericArtifact{name: "golang", version: "1.23.2", filename: "go1.23.2.darwin-arm64.tar.gz.sha256"}, true}, + {"golang", "go1.24rc1.windows-amd64.zip", + genericArtifact{name: "golang", version: "1.24rc1", filename: "go1.24rc1.windows-amd64.zip"}, true}, + {"mirror", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", genericArtifact{}, false}, + } + + for _, tt := range tests { + t.Run(tt.repository+"/"+tt.path, func(t *testing.T) { + got, ok := h.parseArtifact(tt.repository, tt.path) + if ok != tt.ok || got != tt.want { + t.Errorf("parseArtifact(%q, %q) = (%+v, %v), want (%+v, %v)", tt.repository, tt.path, got, ok, tt.want, tt.ok) + } + }) + } +} + func TestGenericHandler_ArtifactPatternIsCachedAndOtherPathsAreNot(t *testing.T) { const ( artifactPath = "/dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz" From 9d0175b1e2b40abb01dfeffa99f7f7e834ddfdf8 Mon Sep 17 00:00:00 2001 From: Giles Westwood <201443+gilesw@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:22:42 +0100 Subject: [PATCH 2/2] Try configured generic_artifacts before built-in patterns The first matching pattern names the package, so an upstream's own patterns now take precedence over overlapping built-ins and keep the package names, cache entries and rules already keyed on them. --- docs/configuration.md | 3 ++- internal/handler/generic.go | 10 ++++++---- internal/handler/generic_test.go | 9 ++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index ce8aa30a..7e94a16a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -371,7 +371,8 @@ the upstream is named: | `https://dl.google.com/go` | `go{version}.{os}-{arch}.tar.gz`, `.zip` and their `.sha256` | the upstream name | Other upstreams can declare their own version-pinned paths in -`generic_artifacts`, which add to any built-in ones. Each pattern is a regular expression that must match the +`generic_artifacts`. These are tried before any built-in ones, so a configured +pattern that overlaps a built-in decides the package name. Each pattern is a regular expression that must match the whole path after `/generic/{name}/`, with `version` and `file` named groups (single path segments) and an optional `name` group for the package name, which otherwise is the upstream name. Matching downloads are cached like diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 954e0911..bf7f0889 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -30,7 +30,7 @@ var githubReleaseAssetPattern = regexp.MustCompile(`^([^/]+)/([^/]+)/releases/do // builtinGenericArtifacts maps well-known download hosts to patterns for // their version-pinned paths. They apply to any generic upstream configured -// with that base URL, before the upstream's own generic_artifacts. +// with that base URL, after the upstream's own generic_artifacts. var builtinGenericArtifacts = map[string][]string{ // dist/{project dirs}/{version}/[binaries/]{file}, e.g. maven/maven-3 or maven/mvnd. "https://archive.apache.org": { @@ -78,15 +78,17 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat artifactPatterns: make(map[string][]*regexp.Regexp), } for name, upstreamURL := range repositories { - upstreamURL = strings.TrimSuffix(upstreamURL, "/") - h.repositories[name] = upstreamURL - h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) + h.repositories[name] = strings.TrimSuffix(upstreamURL, "/") } + // The first matching pattern wins, so configured patterns go first. if len(artifactPatterns) > 0 { for name, patterns := range artifactPatterns[0] { h.addArtifactPatterns(name, patterns) } } + for name, upstreamURL := range h.repositories { + h.addArtifactPatterns(name, builtinGenericArtifacts[upstreamURL]) + } return h } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 1292e8c2..2165b1a6 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -112,7 +112,7 @@ func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { "golang": "https://dl.google.com/go", "mirror": "https://mirror.example.com", }, map[string][]string{ - "apache": {`extra/(?P[^/]+)/(?P[^/]+)`}, + "apache": {`extra/(?P[^/]+)/(?P[^/]+)`, testMavenPattern}, }) tests := []struct { @@ -121,10 +121,13 @@ func TestGenericHandler_BuiltinArtifactPatterns(t *testing.T) { want genericArtifact ok bool }{ + // The configured testMavenPattern overlaps the built-in and takes precedence. {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz", - genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz"}, true}, {"apache", "dist/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz.sha512", - genericArtifact{name: "maven/maven-3", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + genericArtifact{name: "maven", version: "3.9.9", filename: "apache-maven-3.9.9-bin.tar.gz.sha512"}, true}, + {"apache", "dist/maven/maven-4/4.0.0/binaries/apache-maven-4.0.0-bin.tar.gz", + genericArtifact{name: "maven/maven-4", version: "4.0.0", filename: "apache-maven-4.0.0-bin.tar.gz"}, true}, {"apache", "dist/maven/mvnd/1.0.6/maven-mvnd-1.0.6-linux-amd64.tar.gz", genericArtifact{name: "maven/mvnd", version: "1.0.6", filename: "maven-mvnd-1.0.6-linux-amd64.tar.gz"}, true}, {"apache", "dist/maven/maven-3/", genericArtifact{}, false},