Merge pull request #462 from pinguinfuss/issue-409-go-sumdb

Serve the Go checksum database through the proxy

Author: andrew

README.md

@@ -202,6 +202,16 @@ export GOPROXY=http://localhost:8080/go,direct
 
 Or in your shell profile for persistence.
 
+Checksum lookups go through the same URL: the proxy serves `sum.golang.org` at
+`/go/sumdb/sum.golang.org/`, and the `go` command uses it without further
+setup, so clients need no route of their own to `sum.golang.org`. If the
+proxy loses `https://sum.golang.org`, it keeps answering until its cached
+copy of the database's signed tree is older than `metadata_ttl`; after that
+the `go` command contacts the database directly as before. Set
+`upstream.go_sumdbs_offline` on networks where clients cannot do that. Other
+checksum databases are listed under `upstream.go_sumdbs`, see
+[Go checksum database](docs/configuration.md#go-checksum-database).
+
 ### Homebrew
 
 Point Homebrew's JSON API and artifact domain at the proxy:

docs/configuration.md

@@ -237,6 +237,7 @@ Each upstream used by a built-in package route can be set in YAML or JSON under
 | `upstream.cargo_download` | `PROXY_UPSTREAM_CARGO_DOWNLOAD` | `https://static.crates.io/crates` |
 | `upstream.gem` | `PROXY_UPSTREAM_GEM` | `https://rubygems.org` |
 | `upstream.go` | `PROXY_UPSTREAM_GO` | `https://proxy.golang.org` |
+| `upstream.go_sumdbs_offline` | `PROXY_UPSTREAM_GO_SUMDBS_OFFLINE` | `false` |
 | `upstream.hex` | `PROXY_UPSTREAM_HEX` | `https://repo.hex.pm` |
 | `upstream.hex_api` | `PROXY_UPSTREAM_HEX_API` | `https://hex.pm` |
 | `upstream.pub` | `PROXY_UPSTREAM_PUB` | `https://pub.dev` |
@@ -495,6 +496,63 @@ When `upstream.arch` is empty, a single repository named `archlinux` pointing at
 that default. Arch Linux ARM has no default: pick a regional HTTPS mirror, not
 the `mirror.archlinuxarm.org` redirector.
 
+### Go checksum database
+
+```yaml
+upstream:
+
+  # Named Go checksum databases, served at /go/sumdb/{name}/.
+  go_sumdbs:
+    sum.golang.org: "https://sum.golang.org"
+
+  # Keep answering for a database from its cached tree, however old, when
+  # the database cannot be reached. For networks where clients have no
+  # route to the database of their own. Requires cache_metadata.
+  go_sumdbs_offline: false
+```
+
+The `go` command asks `GOPROXY` for the checksum database before contacting it
+directly. The proxy answers for every name listed here and fetches lookups and
+tiles from the database's URL, not from `upstream.go`. Unknown names get 404,
+so the `go` command falls back to contacting that database itself. When
+`upstream.go_sumdbs` is empty, `sum.golang.org` is served; configuring any
+entry replaces that default.
+
+Once the proxy says it serves a database, the `go` command no longer contacts
+that database directly. The proxy therefore only says so when it can fetch the
+database's latest signed tree, or has a fresh one cached (fetched within
+`metadata_ttl`). If it cannot, it answers 404 as for an unknown name, and
+clients go to the database themselves, as they would without the proxy. The
+check waits five seconds at most, and after a failed check the proxy waits a
+minute before it tries the database again.
+
+Set `upstream.go_sumdbs_offline: true` when clients have no route to the
+database of their own. The proxy then keeps answering for it from the cached
+tree for as long as the outage lasts and serves cached lookups and tiles.
+That does not mean every cached module still verifies: the `go` command
+checks each lookup against tiles of the signed tree that came with it, the
+database attaches a different tree to almost every lookup, and a tile no
+client fetched through the proxy before cannot be served during the outage.
+Lookups for modules that were never cached fail as well. Builds whose
+`go.sum` already lists every module do not ask the database at all and are
+not affected. This requires `cache_metadata`.
+
+With `cache_metadata` enabled, lookups, partial hash tiles and the latest
+signed tree follow `metadata_ttl` and are served from cache when the database
+is unreachable. Complete hash tiles never change and are not revalidated once
+cached; a tile of the wrong size is refused and not stored. Data tiles are
+always forwarded and never stored: the `go` command does not fetch them, only
+mirrors and auditors do.
+Without `cache_metadata` the requests are forwarded and nothing is stored.
+
+A 404 from the database keeps its text, which the `go` command prints to
+explain why a version is missing. With `cache_metadata` enabled the text is
+cut at 4 KiB; without it the response is passed on unchanged.
+
+If `upstream.go` points at a module proxy that serves the checksum database
+itself, such as Athens, point the entry there instead, for example
+`sum.golang.org: "https://athens.example/sumdb/sum.golang.org"`.
+
 ### Debian archives
 
 ```yaml

go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/spdx/tools-golang v0.5.7
 	github.com/swaggo/swag v1.16.6
 	gocloud.dev v0.46.0
+	golang.org/x/mod v0.41.0
 	golang.org/x/net v0.59.0
 	golang.org/x/sync v0.23.0
 	google.golang.org/protobuf v1.36.12
@@ -117,7 +118,6 @@ require (
 	go.opentelemetry.io/otel/trace v1.45.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	golang.org/x/crypto v0.57.0 // indirect
-	golang.org/x/mod v0.41.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.42.0 // indirect

internal/config/config.go

@@ -573,6 +573,21 @@ type UpstreamConfig struct {
 	// Default: https://proxy.golang.org
 	Go string `json:"go" yaml:"go"`
 
+	// GoSumDBs maps Go checksum database names to their base URLs, served at
+	// /go/sumdb/{name}/ so the go command reaches the database through
+	// GOPROXY. Unknown names get 404, which makes the go command contact the
+	// database directly.
+	// Default when empty: {"sum.golang.org": "https://sum.golang.org"}.
+	GoSumDBs map[string]string `json:"go_sumdbs" yaml:"go_sumdbs"`
+
+	// GoSumDBsOffline keeps answering for a checksum database from its
+	// cached signed tree, however old, when the database cannot be reached.
+	// Set this when clients have no route to the database of their own;
+	// otherwise the proxy stops answering once the cached tree is older than
+	// metadata_ttl, and the go command contacts the database directly.
+	// Requires cache_metadata. Default: false.
+	GoSumDBsOffline bool `json:"go_sumdbs_offline" yaml:"go_sumdbs_offline"`
+
 	// Hex is the upstream Hex repository URL.
 	// Default: https://repo.hex.pm
 	Hex string `json:"hex" yaml:"hex"`
@@ -773,6 +788,9 @@ func (u *UpstreamConfig) Validate() error {
 	if err := validateNamedUpstreams("upstream.arch", u.Arch); err != nil {
 		return err
 	}
+	if err := validateNamedUpstreams("upstream.go_sumdbs", u.GoSumDBs); err != nil {
+		return err
+	}
 	if err := validateNamedUpstreams("upstream.oci", u.OCI); err != nil {
 		return err
 	}
@@ -813,6 +831,18 @@ func (u *UpstreamConfig) validateGenericArtifacts() error {
 	return nil
 }
 
+// validateUpstream checks the upstream block, including the one setting in
+// it that depends on a top-level option.
+func (c *Config) validateUpstream() error {
+	if err := c.Upstream.Validate(); err != nil {
+		return err
+	}
+	if c.Upstream.GoSumDBsOffline && !c.CacheMetadata {
+		return fmt.Errorf("upstream.go_sumdbs_offline requires cache_metadata")
+	}
+	return nil
+}
+
 // CompileGenericArtifactPattern compiles an upstream.generic_artifacts pattern
 // so that it only matches a whole path, and checks it has the `version` and
 // `file` groups.
@@ -1140,6 +1170,7 @@ func (c *Config) LoadFromEnv() {
 	setEnvBool(&c.Upstream.AllowLoopback, "PROXY_UPSTREAM_ALLOW_LOOPBACK")
 	setEnvString(&c.Upstream.NPM, "PROXY_UPSTREAM_NPM")
 	setEnvBool(&c.Upstream.NPMFullMetadata, "PROXY_UPSTREAM_NPM_FULL_METADATA")
+	setEnvBool(&c.Upstream.GoSumDBsOffline, "PROXY_UPSTREAM_GO_SUMDBS_OFFLINE")
 	setEnvString(&c.Upstream.Cargo, "PROXY_UPSTREAM_CARGO")
 	setEnvString(&c.Upstream.CargoDownload, "PROXY_UPSTREAM_CARGO_DOWNLOAD")
 	setEnvString(&c.Upstream.Gem, "PROXY_UPSTREAM_GEM")
@@ -1315,7 +1346,7 @@ func (c *Config) validateComponents() error {
 		return err
 	}
 
-	if err := c.Upstream.Validate(); err != nil {
+	if err := c.validateUpstream(); err != nil {
 		return err
 	}
 

internal/config/config_test.go

@@ -1612,6 +1612,61 @@ func TestValidateNamedUpstreams(t *testing.T) {
 	}
 }
 
+func TestValidateGoSumDBs(t *testing.T) {
+	tests := []struct {
+		name    string
+		modify  func(*Config)
+		wantErr bool
+	}{
+		{
+			name: "valid Go checksum database",
+			modify: func(cfg *Config) {
+				cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "https://sum.golang.org"}
+			},
+		},
+		{
+			name: "Go checksum database name contains path separator",
+			modify: func(cfg *Config) {
+				cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org/x": "https://sum.golang.org"}
+			},
+			wantErr: true,
+		},
+		{
+			name: "Go checksum database URL is not absolute",
+			modify: func(cfg *Config) {
+				cfg.Upstream.GoSumDBs = map[string]string{"sum.golang.org": "sum.golang.org"}
+			},
+			wantErr: true,
+		},
+		{
+			name: "Go checksum database offline mode requires metadata caching",
+			modify: func(cfg *Config) {
+				cfg.CacheMetadata = false
+				cfg.Upstream.GoSumDBsOffline = true
+			},
+			wantErr: true,
+		},
+		{
+			name: "Go checksum database offline mode with metadata caching",
+			modify: func(cfg *Config) {
+				cfg.CacheMetadata = true
+				cfg.Upstream.GoSumDBsOffline = true
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := Default()
+			tt.modify(cfg)
+			err := cfg.Validate()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Validate() error = %v, wantErr %t", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func TestValidateCacheArtifactsDisabled(t *testing.T) {
 	tests := []struct {
 		name    string

internal/handler/download_test.go

@@ -330,7 +330,6 @@ func TestGoHandler_UpstreamProxy(t *testing.T) {
 		"/golang.org/x/text/@v/v0.14.0.info",
 		"/golang.org/x/text/@v/v0.14.0.mod",
 		"/golang.org/x/text/@latest",
-		"/sumdb/sum.golang.org/lookup/golang.org/x/text@v0.14.0",
 	}
 
 	for _, path := range tests {

internal/handler/go.go

@@ -28,9 +28,12 @@ func init() {
 
 // GoHandler handles Go module proxy protocol requests.
 type GoHandler struct {
-	proxy       *Proxy
-	upstreamURL string
-	proxyURL    string
+	proxy        *Proxy
+	upstreamURL  string
+	proxyURL     string
+	sumDBs       map[string]string
+	sumDBOffline bool
+	sumDBProbes  goSumDBProbes
 }
 
 // NewGoHandler creates a new Go module proxy handler.
@@ -39,6 +42,7 @@ func NewGoHandler(proxy *Proxy, proxyURL string) *GoHandler {
 		proxy:       proxy,
 		upstreamURL: goUpstream,
 		proxyURL:    strings.TrimSuffix(proxyURL, "/"),
+		sumDBs:      map[string]string{defaultGoSumDBName: defaultGoSumDBURL},
 	}
 }
 
@@ -65,9 +69,9 @@ func (h *GoHandler) Routes() http.Handler {
 func (h *GoHandler) handleRequest(w http.ResponseWriter, r *http.Request) {
 	path := strings.TrimPrefix(r.URL.Path, "/")
 
-	// Sumdb requests - proxy through
-	if strings.HasPrefix(path, "sumdb/") {
-		h.proxyUpstream(w, r)
+	// Checksum database requests go to the database, not the module upstream
+	if rest, ok := strings.CutPrefix(path, "sumdb/"); ok {
+		h.handleSumDB(w, r, rest)
 		return
 	}
 
@@ -141,11 +145,6 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul
 	ServeArtifactRequest(w, r, result)
 }
 
-// proxyUpstream forwards a request to proxy.golang.org without caching.
-func (h *GoHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) {
-	h.proxy.ProxyUpstream(w, r, h.upstreamURL+r.URL.Path, nil)
-}
-
 // proxyCached forwards a request with metadata caching.
 func (h *GoHandler) proxyCached(w http.ResponseWriter, r *http.Request, cacheKey string) {
 	h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "golang", cacheKey, "*/*")