From 112e9c89ec0ef37ba98a771114060fbaa1a36350 Mon Sep 17 00:00:00 2001 From: Simon Chrzanowski Date: Sat, 24 Oct 2026 08:02:22 +0200 Subject: [PATCH] Require generic_paths for generic upstreams with credentials A generic upstream forwards any path below /generic/{name}/. The documented mise setup puts a GITHUB_TOKEN on api.github.com to raise the rate limit, so anyone who could reach the proxy could make any GET request with that token, including for private repositories it can read, and the answers went into the shared metadata cache. upstream.git already refuses an upstream with credentials unless it lists its repos. upstream.generic_paths now lists, per generic upstream, the paths it may serve, with the same pattern syntax as git repos. Other paths answer 404 without reaching the upstream, and so does a path holding '%', '?', '#' or a backslash, which the upstream could read as a different path than the one that matched. Config validation refuses a generic upstream that upstream.auth has credentials for unless it has generic_paths, which also covers a github.com token reaching the generic github upstream. Upstreams without credentials keep serving every path. The docs and example config show the paths mise needs and recommend a fine-grained token limited to public repositories. --- README.md | 22 +++++- config.example.yaml | 12 +++ docs/configuration.md | 29 ++++++-- internal/config/config.go | 58 +++++++++++++++ internal/config/config_test.go | 123 +++++++++++++++++++++++++++++++ internal/handler/generic.go | 12 ++- internal/handler/generic_test.go | 44 +++++++++++ internal/server/server.go | 3 +- 8 files changed, 294 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3421dffb..70130a6c 100644 --- a/README.md +++ b/README.md @@ -859,8 +859,26 @@ Release assets are cached permanently after the first download and keep installing while GitHub is down. Tag lookups through `api.github.com` are cached for `metadata_ttl` and served stale during an outage or rate limit. Commit a `mise.lock` and install with `mise install --locked` so pinned -installs need no API call at all. Add a bearer token for `https://api.github.com` -under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. +installs need no API call at all. + +If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for +`https://api.github.com` under `upstream.auth`. Everyone who can reach the +proxy can then make requests with that token, so the proxy requires +`generic_paths` listing the API paths it may serve, and the token should be a +fine-grained one with read-only access to public repositories: + +```yaml +upstream: + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" + auth: + "https://api.github.com": + type: bearer + token: "${GITHUB_TOKEN}" +``` Maven, mvnd, Node and Go downloads are cached the same way once their hosts are configured as generic upstreams; their version-pinned paths are built in diff --git a/config.example.yaml b/config.example.yaml index f7852327..d3b6da49 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -332,6 +332,18 @@ upstream: # github-api: "https://api.github.com" # apache: "https://archive.apache.org" + # Paths a generic upstream may serve, as path.Match patterns for the path + # after /generic/{name}/; a pattern ending in /** also matches everything + # nested below its prefix. Other paths answer 404. Required for an upstream + # that upstream.auth has credentials for, such as a GitHub token for + # api.github.com; use a fine-grained token with read-only access to public + # repositories. + # generic_paths: + # github-api: + # - "repos/*/*/releases" + # - "repos/*/*/releases/**" + # - "repos/*/*/tags" + # Version-pinned paths of generic upstreams, cached immutably like GitHub # release assets. Each pattern must match the whole path after # /generic/{name}/ and have `version` and `file` named groups; an optional diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec69..fc02e781 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -391,6 +391,13 @@ upstream: generic: github: "https://github.com" github-api: "https://api.github.com" + # Paths each upstream may serve. Required for an upstream that + # upstream.auth has credentials for. + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" auth: # Optional: raise the GitHub API rate limit. Scoped to this host only, # so the token is never sent to the object store GitHub redirects to. @@ -400,6 +407,19 @@ upstream: ``` Only configured upstreams are reachable, so this is not an open HTTP proxy. +`generic_paths` limits an upstream to the paths after `/generic/{name}/` that +its patterns match; any other path answers 404. The patterns use the syntax of +`upstream.git` `repos`: `path.Match`, where `*` does not cross a `/`, and a +pattern ending in `/**` also matches every path nested below its prefix. A +path containing `%`, `?`, `#` or a backslash is refused for an upstream with +`generic_paths`. An upstream without `generic_paths` serves every path. + +When `upstream.auth` has credentials for a generic upstream, `generic_paths` +is required and the proxy refuses to start without it: everyone who can reach +the proxy can send requests with those credentials, so list only the paths +they need. For GitHub, use a fine-grained personal access token limited to +public repositories with read-only access; it raises the rate limit like any +other token but cannot read anything private, even through an allowed path. Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is @@ -711,11 +731,10 @@ without `repos` serves every repository on it, and a repository outside `repos` is required: everyone who can reach the proxy can read every repository it serves, so list only those they may read. Per-client permissions are not supported. `upstream.auth` matches credentials by URL, so -they are sent by every route to that host, not only `/git`. With a token for -`https://github.com`, a `generic` upstream at the same URL, such as the one -the mise setup in the README uses, sends it for any path and checks no -`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack` -would list a private repository's refs to anyone. +they are sent by every route to that host, not only `/git`. A `generic` +upstream at the same URL, such as the one the mise setup in the README uses, +checks `generic_paths` instead of `repos`, and needs them too; keep Git paths +like `{owner}/{repo}.git/info/refs` out of its patterns. Every request for a repository's refs checks the upstream's branches and tags, and fetches only when they changed, so a clone right after a push sees the new diff --git a/internal/config/config.go b/internal/config/config.go index d7904161..a6606994 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -785,6 +785,16 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // GenericPaths maps a Generic upstream name to the paths it may serve, + // as patterns for the request path after /generic/{name}/ with the same + // syntax as GitUpstream.Repos: path.Match, where a pattern ending in + // "/**" also matches every path nested below its prefix. Other paths + // answer 404. Empty serves every path. Required when upstream.auth has + // credentials for the upstream, since everyone who can reach the proxy + // can use them for any path otherwise. + // Example: {"github-api": ["repos/*/*/releases/**"]}. + GenericPaths map[string][]string `json:"generic_paths" yaml:"generic_paths"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} // from a local cache of bare repositories. See GitUpstream. // Example: {"github": {"url": "https://github.com"}}. @@ -863,6 +873,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGenericPaths(); err != nil { + return err + } if err := u.validateGitUpstreams(); err != nil { return err } @@ -902,6 +915,51 @@ func (u *UpstreamConfig) validateGenericArtifacts() error { return nil } +// validateGenericPaths checks upstream.generic_paths patterns. An upstream that +// upstream.auth has credentials for must list its paths, so configuring a +// token does not let everyone who can reach the proxy send any request with +// it, mirroring the repos rule for upstream.git. +func (u *UpstreamConfig) validateGenericPaths() error { + for name, patterns := range u.GenericPaths { + if _, found := u.Generic[name]; !found { + return fmt.Errorf("invalid upstream.generic_paths name %q: no upstream.generic entry", name) + } + for _, pattern := range patterns { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid upstream.generic_paths.%s pattern %q", name, pattern) + } + } + } + for name, upstreamURL := range u.Generic { + if len(u.GenericPaths[name]) == 0 && u.HasAuthWithin(upstreamURL) { + return fmt.Errorf("upstream.generic.%s needs upstream.generic_paths.%s: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could send any request with them", name, name, upstreamURL) + } + } + return nil +} + +// GenericPathAllowed reports whether p, a request path after +// /generic/{name}/, matches one of an upstream's upstream.generic_paths +// patterns. No patterns allow every path. With patterns, a path holding a +// character the upstream could read as the end of the path or as a different +// path ('%', '?', '#' or a backslash) is refused, so the path that matched is +// the one the upstream sees. +func GenericPathAllowed(patterns []string, p string) bool { + if len(patterns) == 0 { + return true + } + if strings.ContainsAny(p, `%?#\`) { + return false + } + for _, pattern := range patterns { + if matchRepoPattern(pattern, p) { + return true + } + } + return false +} + // validateUpstream checks the upstream block, including the one setting in // it that depends on a top-level option. func (c *Config) validateUpstream() error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 41f414cd..7590a64f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1929,3 +1929,126 @@ func TestValidateFreeBSDUpstreams(t *testing.T) { }) } } + +func TestValidateGenericPaths(t *testing.T) { + githubAPI := map[string]string{"github-api": "https://api.github.com"} + token := map[string]AuthConfig{"https://api.github.com": {Type: "bearer", Token: "t"}} + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Generic: githubAPI}}, + {name: "credentials without paths", upstream: UpstreamConfig{Generic: githubAPI, Auth: token}, wantErr: true}, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://api.github.com/repos/acme": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with paths", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"repos/*/*/releases/**"}}, + Auth: token, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "paths for an unknown upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github": {"*/*/releases/**"}}, + }, + wantErr: true, + }, + { + name: "bad pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"["}}, + }, + wantErr: true, + }, + { + name: "empty pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {""}}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGenericPathAllowed(t *testing.T) { + patterns := []string{"repos/*/*/releases/**", "rate_limit"} + tests := []struct { + path string + want bool + }{ + {"repos/jqlang/jq/releases/tags/jq-1.7.1", true}, + {"repos/jqlang/jq/releases/latest", true}, + {"rate_limit", true}, + {"repos/jqlang/jq/releases", false}, + {"repos/acme/private/contents/secret.txt", false}, + {"user/repos", false}, + {"repos/jqlang/jq/releases/%2e%2e", false}, + {"repos/jqlang/jq?/releases/x", false}, + {"repos/jqlang/jq/releases/x#y", false}, + {`repos/jqlang/jq/releases/x\y`, false}, + } + for _, tt := range tests { + if got := GenericPathAllowed(patterns, tt.path); got != tt.want { + t.Errorf("GenericPathAllowed(%q) = %v, want %v", tt.path, got, tt.want) + } + } + if !GenericPathAllowed(nil, "user/repos") { + t.Error("GenericPathAllowed without patterns refused a path") + } +} + +func TestLoadGenericPaths(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + generic: + github-api: https://api.github.com + generic_paths: + github-api: + - "repos/*/*/releases/**" + auth: + "https://api.github.com": + type: bearer + token: t +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Upstream.GenericPaths["github-api"]; len(got) != 1 || got[0] != "repos/*/*/releases/**" { + t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) + } +} diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 700e53e2..0e08140e 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -71,6 +71,9 @@ type GenericHandler struct { proxy *Proxy repositories map[string]string artifactPatterns map[string][]*regexp.Regexp + // paths holds each upstream's upstream.generic_paths patterns; an + // upstream without any serves every path. + paths map[string][]string } // NewGenericHandler creates a generic HTTP download proxy handler. @@ -97,6 +100,13 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat return h } +// WithPaths limits upstreams to the paths their upstream.generic_paths +// patterns match. Any other path answers 404. +func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler { + h.paths = paths + return h +} + func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { for _, pattern := range patterns { re, err := config.CompileGenericArtifactPattern(pattern) @@ -131,7 +141,7 @@ func (h *GenericHandler) Routes() http.Handler { return } upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found { + if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) { http.NotFound(w, r) return } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 424eafb4..a74be8c8 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -646,3 +646,47 @@ func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) { t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body) } } + +func TestGenericHandler_ServesOnlyConfiguredPaths(t *testing.T) { + var requested []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested = append(requested, r.URL.RequestURI()) + _, _ = io.WriteString(w, "from upstream "+r.URL.Path) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL, "open": upstream.URL}). + WithPaths(map[string][]string{"github-api": {"repos/*/*/releases/**", "rate_limit"}}) + + for _, target := range []string{ + "/github-api/repos/jqlang/jq/releases/tags/jq-1.7.1", + "/github-api/rate_limit", + "/open/user/repos", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusOK { + t.Errorf("GET %s: status = %d, want 200: %s", target, w.Code, w.Body.String()) + } + } + + served := len(requested) + for _, target := range []string{ + "/github-api/user/repos", + "/github-api/repos/acme/private/contents/secret.txt", + "/github-api/repos/jqlang/jq/releases", + "/github-api/repos/jqlang/jq%3F/releases/x", + "/github-api/repos/jqlang/jq/releases/%2575ser", + "/github-api/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("GET %s: status = %d, want 404", target, w.Code) + } + } + if len(requested) != served { + t.Errorf("upstream saw %v for paths outside generic_paths", requested[served:]) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf72..d64e5cfb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -493,7 +493,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.DebianRepositories, ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) - genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts). + WithPaths(s.cfg.Upstream.GenericPaths) r = r.With(sandboxUpstreamContent, rejectQueryInPath, rejectDotSegments) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))