diff --git a/README.md b/README.md index 3421dffb..70130a6c 100644 --- a/README.md +++ b/README.md @@ -859,8 +859,26 @@ Release assets are cached permanently after the first download and keep installing while GitHub is down. Tag lookups through `api.github.com` are cached for `metadata_ttl` and served stale during an outage or rate limit. Commit a `mise.lock` and install with `mise install --locked` so pinned -installs need no API call at all. Add a bearer token for `https://api.github.com` -under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit. +installs need no API call at all. + +If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for +`https://api.github.com` under `upstream.auth`. Everyone who can reach the +proxy can then make requests with that token, so the proxy requires +`generic_paths` listing the API paths it may serve, and the token should be a +fine-grained one with read-only access to public repositories: + +```yaml +upstream: + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" + auth: + "https://api.github.com": + type: bearer + token: "${GITHUB_TOKEN}" +``` Maven, mvnd, Node and Go downloads are cached the same way once their hosts are configured as generic upstreams; their version-pinned paths are built in diff --git a/config.example.yaml b/config.example.yaml index f7852327..d3b6da49 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -332,6 +332,18 @@ upstream: # github-api: "https://api.github.com" # apache: "https://archive.apache.org" + # Paths a generic upstream may serve, as path.Match patterns for the path + # after /generic/{name}/; a pattern ending in /** also matches everything + # nested below its prefix. Other paths answer 404. Required for an upstream + # that upstream.auth has credentials for, such as a GitHub token for + # api.github.com; use a fine-grained token with read-only access to public + # repositories. + # generic_paths: + # github-api: + # - "repos/*/*/releases" + # - "repos/*/*/releases/**" + # - "repos/*/*/tags" + # Version-pinned paths of generic upstreams, cached immutably like GitHub # release assets. Each pattern must match the whole path after # /generic/{name}/ and have `version` and `file` named groups; an optional diff --git a/docs/configuration.md b/docs/configuration.md index fa77ec69..fc02e781 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -391,6 +391,13 @@ upstream: generic: github: "https://github.com" github-api: "https://api.github.com" + # Paths each upstream may serve. Required for an upstream that + # upstream.auth has credentials for. + generic_paths: + github-api: + - "repos/*/*/releases" + - "repos/*/*/releases/**" + - "repos/*/*/tags" auth: # Optional: raise the GitHub API rate limit. Scoped to this host only, # so the token is never sent to the object store GitHub redirects to. @@ -400,6 +407,19 @@ upstream: ``` Only configured upstreams are reachable, so this is not an open HTTP proxy. +`generic_paths` limits an upstream to the paths after `/generic/{name}/` that +its patterns match; any other path answers 404. The patterns use the syntax of +`upstream.git` `repos`: `path.Match`, where `*` does not cross a `/`, and a +pattern ending in `/**` also matches every path nested below its prefix. A +path containing `%`, `?`, `#` or a backslash is refused for an upstream with +`generic_paths`. An upstream without `generic_paths` serves every path. + +When `upstream.auth` has credentials for a generic upstream, `generic_paths` +is required and the proxy refuses to start without it: everyone who can reach +the proxy can send requests with those credentials, so list only the paths +they need. For GitHub, use a fine-grained personal access token limited to +public repositories with read-only access; it raises the rate limit like any +other token but cannot read anything private, even through an allowed path. Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are version-pinned GitHub release assets: they are stored in the artifact cache and served from it without revalidation, including while the upstream is @@ -711,11 +731,10 @@ without `repos` serves every repository on it, and a repository outside `repos` is required: everyone who can reach the proxy can read every repository it serves, so list only those they may read. Per-client permissions are not supported. `upstream.auth` matches credentials by URL, so -they are sent by every route to that host, not only `/git`. With a token for -`https://github.com`, a `generic` upstream at the same URL, such as the one -the mise setup in the README uses, sends it for any path and checks no -`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack` -would list a private repository's refs to anyone. +they are sent by every route to that host, not only `/git`. A `generic` +upstream at the same URL, such as the one the mise setup in the README uses, +checks `generic_paths` instead of `repos`, and needs them too; keep Git paths +like `{owner}/{repo}.git/info/refs` out of its patterns. Every request for a repository's refs checks the upstream's branches and tags, and fetches only when they changed, so a clone right after a push sees the new diff --git a/internal/config/config.go b/internal/config/config.go index d7904161..a6606994 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -785,6 +785,16 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // GenericPaths maps a Generic upstream name to the paths it may serve, + // as patterns for the request path after /generic/{name}/ with the same + // syntax as GitUpstream.Repos: path.Match, where a pattern ending in + // "/**" also matches every path nested below its prefix. Other paths + // answer 404. Empty serves every path. Required when upstream.auth has + // credentials for the upstream, since everyone who can reach the proxy + // can use them for any path otherwise. + // Example: {"github-api": ["repos/*/*/releases/**"]}. + GenericPaths map[string][]string `json:"generic_paths" yaml:"generic_paths"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} // from a local cache of bare repositories. See GitUpstream. // Example: {"github": {"url": "https://github.com"}}. @@ -863,6 +873,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGenericPaths(); err != nil { + return err + } if err := u.validateGitUpstreams(); err != nil { return err } @@ -902,6 +915,51 @@ func (u *UpstreamConfig) validateGenericArtifacts() error { return nil } +// validateGenericPaths checks upstream.generic_paths patterns. An upstream that +// upstream.auth has credentials for must list its paths, so configuring a +// token does not let everyone who can reach the proxy send any request with +// it, mirroring the repos rule for upstream.git. +func (u *UpstreamConfig) validateGenericPaths() error { + for name, patterns := range u.GenericPaths { + if _, found := u.Generic[name]; !found { + return fmt.Errorf("invalid upstream.generic_paths name %q: no upstream.generic entry", name) + } + for _, pattern := range patterns { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid upstream.generic_paths.%s pattern %q", name, pattern) + } + } + } + for name, upstreamURL := range u.Generic { + if len(u.GenericPaths[name]) == 0 && u.HasAuthWithin(upstreamURL) { + return fmt.Errorf("upstream.generic.%s needs upstream.generic_paths.%s: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could send any request with them", name, name, upstreamURL) + } + } + return nil +} + +// GenericPathAllowed reports whether p, a request path after +// /generic/{name}/, matches one of an upstream's upstream.generic_paths +// patterns. No patterns allow every path. With patterns, a path holding a +// character the upstream could read as the end of the path or as a different +// path ('%', '?', '#' or a backslash) is refused, so the path that matched is +// the one the upstream sees. +func GenericPathAllowed(patterns []string, p string) bool { + if len(patterns) == 0 { + return true + } + if strings.ContainsAny(p, `%?#\`) { + return false + } + for _, pattern := range patterns { + if matchRepoPattern(pattern, p) { + return true + } + } + return false +} + // validateUpstream checks the upstream block, including the one setting in // it that depends on a top-level option. func (c *Config) validateUpstream() error { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 41f414cd..7590a64f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1929,3 +1929,126 @@ func TestValidateFreeBSDUpstreams(t *testing.T) { }) } } + +func TestValidateGenericPaths(t *testing.T) { + githubAPI := map[string]string{"github-api": "https://api.github.com"} + token := map[string]AuthConfig{"https://api.github.com": {Type: "bearer", Token: "t"}} + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Generic: githubAPI}}, + {name: "credentials without paths", upstream: UpstreamConfig{Generic: githubAPI, Auth: token}, wantErr: true}, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://api.github.com/repos/acme": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with paths", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"repos/*/*/releases/**"}}, + Auth: token, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Generic: githubAPI, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "paths for an unknown upstream", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github": {"*/*/releases/**"}}, + }, + wantErr: true, + }, + { + name: "bad pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {"["}}, + }, + wantErr: true, + }, + { + name: "empty pattern", + upstream: UpstreamConfig{ + Generic: githubAPI, + GenericPaths: map[string][]string{"github-api": {""}}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGenericPathAllowed(t *testing.T) { + patterns := []string{"repos/*/*/releases/**", "rate_limit"} + tests := []struct { + path string + want bool + }{ + {"repos/jqlang/jq/releases/tags/jq-1.7.1", true}, + {"repos/jqlang/jq/releases/latest", true}, + {"rate_limit", true}, + {"repos/jqlang/jq/releases", false}, + {"repos/acme/private/contents/secret.txt", false}, + {"user/repos", false}, + {"repos/jqlang/jq/releases/%2e%2e", false}, + {"repos/jqlang/jq?/releases/x", false}, + {"repos/jqlang/jq/releases/x#y", false}, + {`repos/jqlang/jq/releases/x\y`, false}, + } + for _, tt := range tests { + if got := GenericPathAllowed(patterns, tt.path); got != tt.want { + t.Errorf("GenericPathAllowed(%q) = %v, want %v", tt.path, got, tt.want) + } + } + if !GenericPathAllowed(nil, "user/repos") { + t.Error("GenericPathAllowed without patterns refused a path") + } +} + +func TestLoadGenericPaths(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + generic: + github-api: https://api.github.com + generic_paths: + github-api: + - "repos/*/*/releases/**" + auth: + "https://api.github.com": + type: bearer + token: t +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if got := cfg.Upstream.GenericPaths["github-api"]; len(got) != 1 || got[0] != "repos/*/*/releases/**" { + t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths) + } +} diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 700e53e2..0e08140e 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -71,6 +71,9 @@ type GenericHandler struct { proxy *Proxy repositories map[string]string artifactPatterns map[string][]*regexp.Regexp + // paths holds each upstream's upstream.generic_paths patterns; an + // upstream without any serves every path. + paths map[string][]string } // NewGenericHandler creates a generic HTTP download proxy handler. @@ -97,6 +100,13 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat return h } +// WithPaths limits upstreams to the paths their upstream.generic_paths +// patterns match. Any other path answers 404. +func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler { + h.paths = paths + return h +} + func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) { for _, pattern := range patterns { re, err := config.CompileGenericArtifactPattern(pattern) @@ -131,7 +141,7 @@ func (h *GenericHandler) Routes() http.Handler { return } upstreamURL, found := h.repositories[repository] - if !ok || rest == "" || !found { + if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) { http.NotFound(w, r) return } diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 424eafb4..a74be8c8 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -646,3 +646,47 @@ func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) { t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body) } } + +func TestGenericHandler_ServesOnlyConfiguredPaths(t *testing.T) { + var requested []string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested = append(requested, r.URL.RequestURI()) + _, _ = io.WriteString(w, "from upstream "+r.URL.Path) + })) + t.Cleanup(upstream.Close) + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL, "open": upstream.URL}). + WithPaths(map[string][]string{"github-api": {"repos/*/*/releases/**", "rate_limit"}}) + + for _, target := range []string{ + "/github-api/repos/jqlang/jq/releases/tags/jq-1.7.1", + "/github-api/rate_limit", + "/open/user/repos", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusOK { + t.Errorf("GET %s: status = %d, want 200: %s", target, w.Code, w.Body.String()) + } + } + + served := len(requested) + for _, target := range []string{ + "/github-api/user/repos", + "/github-api/repos/acme/private/contents/secret.txt", + "/github-api/repos/jqlang/jq/releases", + "/github-api/repos/jqlang/jq%3F/releases/x", + "/github-api/repos/jqlang/jq/releases/%2575ser", + "/github-api/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64", + } { + if w := serveGenericRequest(h, target); w.Code != http.StatusNotFound { + t.Errorf("GET %s: status = %d, want 404", target, w.Code) + } + } + if len(requested) != served { + t.Errorf("upstream saw %v for paths outside generic_paths", requested[served:]) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 5b0ebf72..d64e5cfb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -493,7 +493,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.DebianRepositories, ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) - genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts) + genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts). + WithPaths(s.cfg.Upstream.GenericPaths) r = r.With(sandboxUpstreamContent, rejectQueryInPath, rejectDotSegments) r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))