Require generic_paths for generic upstreams with credentials

README.md

@@ -859,8 +859,26 @@ Release assets are cached permanently after the first download and keep
 installing while GitHub is down. Tag lookups through `api.github.com` are
 cached for `metadata_ttl` and served stale during an outage or rate limit.
 Commit a `mise.lock` and install with `mise install --locked` so pinned
-installs need no API call at all. Add a bearer token for `https://api.github.com`
-under `upstream.auth` if the fleet exceeds GitHub's anonymous rate limit.
+installs need no API call at all.
+
+If the fleet exceeds GitHub's anonymous rate limit, add a bearer token for
+`https://api.github.com` under `upstream.auth`. Everyone who can reach the
+proxy can then make requests with that token, so the proxy requires
+`generic_paths` listing the API paths it may serve, and the token should be a
+fine-grained one with read-only access to public repositories:
+
+```yaml
+upstream:
+  generic_paths:
+    github-api:
+      - "repos/*/*/releases"
+      - "repos/*/*/releases/**"
+      - "repos/*/*/tags"
+  auth:
+    "https://api.github.com":
+      type: bearer
+      token: "${GITHUB_TOKEN}"
+```
 
 Maven, mvnd, Node and Go downloads are cached the same way once their hosts
 are configured as generic upstreams; their version-pinned paths are built in

config.example.yaml

@@ -332,6 +332,18 @@ upstream:
   #   github-api: "https://api.github.com"
   #   apache: "https://archive.apache.org"
 
+  # Paths a generic upstream may serve, as path.Match patterns for the path
+  # after /generic/{name}/; a pattern ending in /** also matches everything
+  # nested below its prefix. Other paths answer 404. Required for an upstream
+  # that upstream.auth has credentials for, such as a GitHub token for
+  # api.github.com; use a fine-grained token with read-only access to public
+  # repositories.
+  # generic_paths:
+  #   github-api:
+  #     - "repos/*/*/releases"
+  #     - "repos/*/*/releases/**"
+  #     - "repos/*/*/tags"
+
   # Version-pinned paths of generic upstreams, cached immutably like GitHub
   # release assets. Each pattern must match the whole path after
   # /generic/{name}/ and have `version` and `file` named groups; an optional

docs/configuration.md

@@ -391,6 +391,13 @@ upstream:
   generic:
     github: "https://github.com"
     github-api: "https://api.github.com"
+  # Paths each upstream may serve. Required for an upstream that
+  # upstream.auth has credentials for.
+  generic_paths:
+    github-api:
+      - "repos/*/*/releases"
+      - "repos/*/*/releases/**"
+      - "repos/*/*/tags"
   auth:
     # Optional: raise the GitHub API rate limit. Scoped to this host only,
     # so the token is never sent to the object store GitHub redirects to.
@@ -400,6 +407,19 @@ upstream:
 ```
 
 Only configured upstreams are reachable, so this is not an open HTTP proxy.
+`generic_paths` limits an upstream to the paths after `/generic/{name}/` that
+its patterns match; any other path answers 404. The patterns use the syntax of
+`upstream.git` `repos`: `path.Match`, where `*` does not cross a `/`, and a
+pattern ending in `/**` also matches every path nested below its prefix. A
+path containing `%`, `?`, `#` or a backslash is refused for an upstream with
+`generic_paths`. An upstream without `generic_paths` serves every path.
+
+When `upstream.auth` has credentials for a generic upstream, `generic_paths`
+is required and the proxy refuses to start without it: everyone who can reach
+the proxy can send requests with those credentials, so list only the paths
+they need. For GitHub, use a fine-grained personal access token limited to
+public repositories with read-only access; it raises the rate limit like any
+other token but cannot read anything private, even through an allowed path.
 Paths shaped like `{owner}/{repo}/releases/download/{tag}/{asset}` are
 version-pinned GitHub release assets: they are stored in the artifact cache
 and served from it without revalidation, including while the upstream is
@@ -711,11 +731,10 @@ without `repos` serves every repository on it, and a repository outside
 `repos` is required: everyone who can reach the proxy can read every
 repository it serves, so list only those they may read. Per-client
 permissions are not supported. `upstream.auth` matches credentials by URL, so
-they are sent by every route to that host, not only `/git`. With a token for
-`https://github.com`, a `generic` upstream at the same URL, such as the one
-the mise setup in the README uses, sends it for any path and checks no
-`repos`, so `/generic/github/acme/private.git/info/refs?service=git-upload-pack`
-would list a private repository's refs to anyone.
+they are sent by every route to that host, not only `/git`. A `generic`
+upstream at the same URL, such as the one the mise setup in the README uses,
+checks `generic_paths` instead of `repos`, and needs them too; keep Git paths
+like `{owner}/{repo}.git/info/refs` out of its patterns.
 
 Every request for a repository's refs checks the upstream's branches and tags,
 and fetches only when they changed, so a clone right after a push sees the new

internal/config/config.go

@@ -785,6 +785,16 @@ type UpstreamConfig struct {
 	// Example: {"apache": ["dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)"]}.
 	GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"`
 
+	// GenericPaths maps a Generic upstream name to the paths it may serve,
+	// as patterns for the request path after /generic/{name}/ with the same
+	// syntax as GitUpstream.Repos: path.Match, where a pattern ending in
+	// "/**" also matches every path nested below its prefix. Other paths
+	// answer 404. Empty serves every path. Required when upstream.auth has
+	// credentials for the upstream, since everyone who can reach the proxy
+	// can use them for any path otherwise.
+	// Example: {"github-api": ["repos/*/*/releases/**"]}.
+	GenericPaths map[string][]string `json:"generic_paths" yaml:"generic_paths"`
+
 	// Git maps names to Git servers, served read-only at /git/{name}/{repo}
 	// from a local cache of bare repositories. See GitUpstream.
 	// Example: {"github": {"url": "https://github.com"}}.
@@ -863,6 +873,9 @@ func (u *UpstreamConfig) Validate() error {
 	if err := u.validateGenericArtifacts(); err != nil {
 		return err
 	}
+	if err := u.validateGenericPaths(); err != nil {
+		return err
+	}
 	if err := u.validateGitUpstreams(); err != nil {
 		return err
 	}
@@ -902,6 +915,51 @@ func (u *UpstreamConfig) validateGenericArtifacts() error {
 	return nil
 }
 
+// validateGenericPaths checks upstream.generic_paths patterns. An upstream that
+// upstream.auth has credentials for must list its paths, so configuring a
+// token does not let everyone who can reach the proxy send any request with
+// it, mirroring the repos rule for upstream.git.
+func (u *UpstreamConfig) validateGenericPaths() error {
+	for name, patterns := range u.GenericPaths {
+		if _, found := u.Generic[name]; !found {
+			return fmt.Errorf("invalid upstream.generic_paths name %q: no upstream.generic entry", name)
+		}
+		for _, pattern := range patterns {
+			if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil {
+				return fmt.Errorf("invalid upstream.generic_paths.%s pattern %q", name, pattern)
+			}
+		}
+	}
+	for name, upstreamURL := range u.Generic {
+		if len(u.GenericPaths[name]) == 0 && u.HasAuthWithin(upstreamURL) {
+			return fmt.Errorf("upstream.generic.%s needs upstream.generic_paths.%s: upstream.auth has credentials for %s, "+
+				"and everyone who can reach the proxy could send any request with them", name, name, upstreamURL)
+		}
+	}
+	return nil
+}
+
+// GenericPathAllowed reports whether p, a request path after
+// /generic/{name}/, matches one of an upstream's upstream.generic_paths
+// patterns. No patterns allow every path. With patterns, a path holding a
+// character the upstream could read as the end of the path or as a different
+// path ('%', '?', '#' or a backslash) is refused, so the path that matched is
+// the one the upstream sees.
+func GenericPathAllowed(patterns []string, p string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+	if strings.ContainsAny(p, `%?#\`) {
+		return false
+	}
+	for _, pattern := range patterns {
+		if matchRepoPattern(pattern, p) {
+			return true
+		}
+	}
+	return false
+}
+
 // validateUpstream checks the upstream block, including the one setting in
 // it that depends on a top-level option.
 func (c *Config) validateUpstream() error {

internal/config/config_test.go

@@ -1929,3 +1929,126 @@ func TestValidateFreeBSDUpstreams(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateGenericPaths(t *testing.T) {
+	githubAPI := map[string]string{"github-api": "https://api.github.com"}
+	token := map[string]AuthConfig{"https://api.github.com": {Type: "bearer", Token: "t"}}
+	tests := []struct {
+		name     string
+		upstream UpstreamConfig
+		wantErr  bool
+	}{
+		{name: "public", upstream: UpstreamConfig{Generic: githubAPI}},
+		{name: "credentials without paths", upstream: UpstreamConfig{Generic: githubAPI, Auth: token}, wantErr: true},
+		{
+			name: "credentials for a path under the upstream",
+			upstream: UpstreamConfig{
+				Generic: githubAPI,
+				Auth:    map[string]AuthConfig{"https://api.github.com/repos/acme": {Type: "bearer", Token: "t"}},
+			},
+			wantErr: true,
+		},
+		{
+			name: "credentials with paths",
+			upstream: UpstreamConfig{
+				Generic:      githubAPI,
+				GenericPaths: map[string][]string{"github-api": {"repos/*/*/releases/**"}},
+				Auth:         token,
+			},
+		},
+		{
+			name: "credentials for another host",
+			upstream: UpstreamConfig{
+				Generic: githubAPI,
+				Auth:    map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}},
+			},
+		},
+		{
+			name: "paths for an unknown upstream",
+			upstream: UpstreamConfig{
+				Generic:      githubAPI,
+				GenericPaths: map[string][]string{"github": {"*/*/releases/**"}},
+			},
+			wantErr: true,
+		},
+		{
+			name: "bad pattern",
+			upstream: UpstreamConfig{
+				Generic:      githubAPI,
+				GenericPaths: map[string][]string{"github-api": {"["}},
+			},
+			wantErr: true,
+		},
+		{
+			name: "empty pattern",
+			upstream: UpstreamConfig{
+				Generic:      githubAPI,
+				GenericPaths: map[string][]string{"github-api": {""}},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := tt.upstream.Validate(); (err != nil) != tt.wantErr {
+				t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestGenericPathAllowed(t *testing.T) {
+	patterns := []string{"repos/*/*/releases/**", "rate_limit"}
+	tests := []struct {
+		path string
+		want bool
+	}{
+		{"repos/jqlang/jq/releases/tags/jq-1.7.1", true},
+		{"repos/jqlang/jq/releases/latest", true},
+		{"rate_limit", true},
+		{"repos/jqlang/jq/releases", false},
+		{"repos/acme/private/contents/secret.txt", false},
+		{"user/repos", false},
+		{"repos/jqlang/jq/releases/%2e%2e", false},
+		{"repos/jqlang/jq?/releases/x", false},
+		{"repos/jqlang/jq/releases/x#y", false},
+		{`repos/jqlang/jq/releases/x\y`, false},
+	}
+	for _, tt := range tests {
+		if got := GenericPathAllowed(patterns, tt.path); got != tt.want {
+			t.Errorf("GenericPathAllowed(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+	if !GenericPathAllowed(nil, "user/repos") {
+		t.Error("GenericPathAllowed without patterns refused a path")
+	}
+}
+
+func TestLoadGenericPaths(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.yaml")
+	data := []byte(`
+upstream:
+  generic:
+    github-api: https://api.github.com
+  generic_paths:
+    github-api:
+      - "repos/*/*/releases/**"
+  auth:
+    "https://api.github.com":
+      type: bearer
+      token: t
+`)
+	if err := os.WriteFile(path, data, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	cfg, err := Load(path)
+	if err != nil {
+		t.Fatalf("Load: %v", err)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate: %v", err)
+	}
+	if got := cfg.Upstream.GenericPaths["github-api"]; len(got) != 1 || got[0] != "repos/*/*/releases/**" {
+		t.Errorf("generic_paths = %v", cfg.Upstream.GenericPaths)
+	}
+}

internal/handler/generic.go

@@ -71,6 +71,9 @@ type GenericHandler struct {
 	proxy            *Proxy
 	repositories     map[string]string
 	artifactPatterns map[string][]*regexp.Regexp
+	// paths holds each upstream's upstream.generic_paths patterns; an
+	// upstream without any serves every path.
+	paths map[string][]string
 }
 
 // NewGenericHandler creates a generic HTTP download proxy handler.
@@ -97,6 +100,13 @@ func NewGenericHandler(proxy *Proxy, repositories map[string]string, artifactPat
 	return h
 }
 
+// WithPaths limits upstreams to the paths their upstream.generic_paths
+// patterns match. Any other path answers 404.
+func (h *GenericHandler) WithPaths(paths map[string][]string) *GenericHandler {
+	h.paths = paths
+	return h
+}
+
 func (h *GenericHandler) addArtifactPatterns(name string, patterns []string) {
 	for _, pattern := range patterns {
 		re, err := config.CompileGenericArtifactPattern(pattern)
@@ -131,7 +141,7 @@ func (h *GenericHandler) Routes() http.Handler {
 			return
 		}
 		upstreamURL, found := h.repositories[repository]
-		if !ok || rest == "" || !found {
+		if !ok || rest == "" || !found || !config.GenericPathAllowed(h.paths[repository], rest) {
 			http.NotFound(w, r)
 			return
 		}

internal/handler/generic_test.go

@@ -646,3 +646,47 @@ func TestGenericHandler_UpstreamNamedHostedWhileHostingIsOff(t *testing.T) {
 		t.Errorf("status = %d, body %q, want the upstream response", w.Code, w.Body)
 	}
 }
+
+func TestGenericHandler_ServesOnlyConfiguredPaths(t *testing.T) {
+	var requested []string
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requested = append(requested, r.URL.RequestURI())
+		_, _ = io.WriteString(w, "from upstream "+r.URL.Path)
+	}))
+	t.Cleanup(upstream.Close)
+	proxy, _, _, _ := setupTestProxy(t)
+	proxy.HTTPClient = upstream.Client()
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
+	proxy.Fetcher = fetcher
+	t.Cleanup(func() { _ = fetcher.Close() })
+
+	h := NewGenericHandler(proxy, map[string]string{"github-api": upstream.URL, "open": upstream.URL}).
+		WithPaths(map[string][]string{"github-api": {"repos/*/*/releases/**", "rate_limit"}})
+
+	for _, target := range []string{
+		"/github-api/repos/jqlang/jq/releases/tags/jq-1.7.1",
+		"/github-api/rate_limit",
+		"/open/user/repos",
+	} {
+		if w := serveGenericRequest(h, target); w.Code != http.StatusOK {
+			t.Errorf("GET %s: status = %d, want 200: %s", target, w.Code, w.Body.String())
+		}
+	}
+
+	served := len(requested)
+	for _, target := range []string{
+		"/github-api/user/repos",
+		"/github-api/repos/acme/private/contents/secret.txt",
+		"/github-api/repos/jqlang/jq/releases",
+		"/github-api/repos/jqlang/jq%3F/releases/x",
+		"/github-api/repos/jqlang/jq/releases/%2575ser",
+		"/github-api/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64",
+	} {
+		if w := serveGenericRequest(h, target); w.Code != http.StatusNotFound {
+			t.Errorf("GET %s: status = %d, want 404", target, w.Code)
+		}
+	}
+	if len(requested) != served {
+		t.Errorf("upstream saw %v for paths outside generic_paths", requested[served:])
+	}
+}

internal/server/server.go

@@ -493,7 +493,8 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) {
 		s.cfg.Upstream.DebianRepositories,
 	)
 	rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM)
-	genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts)
+	genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic, s.cfg.Upstream.GenericArtifacts).
+		WithPaths(s.cfg.Upstream.GenericPaths)
 
 	r = r.With(sandboxUpstreamContent, rejectQueryInPath, rejectDotSegments)
 	r.Mount("/npm", http.StripPrefix("/npm", npmHandler.Routes()))