Extract portable npm metadata transformations

docs/architecture.md

@@ -34,7 +34,7 @@ The proxy is a caching HTTP server that sits between package manager clients and
 1. Client requests `GET /npm/lodash`
 2. NPMHandler receives request
 3. Handler fetches metadata from upstream `registry.npmjs.org/lodash`
-4. Handler rewrites tarball URLs in metadata to point at proxy
+4. Handler rewrites tarball URLs in metadata to point at proxy, and filters versions withheld by cooldown or the denylist (`internal/metadata/npm`)
 5. Handler returns modified metadata to client
 
 Metadata is not cached - always fetched fresh. This ensures clients see new versions immediately.
@@ -294,7 +294,7 @@ HTTP protocol handlers for each registry type.
 **NPMHandler:**
 - `handlePackageMetadata()` - Proxy + rewrite metadata
 - `handleDownload()` - Serve cached artifact
-- Rewrites tarball URLs to point at proxy
+- Fetches, caches and serves packuments, and caches their rewrites; the rewrite itself is `internal/metadata/npm`
 
 **CargoHandler:**
 - `handleConfig()` - Return registry config
@@ -310,6 +310,14 @@ HTTP protocol handlers for each registry type.
 - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses
 - Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache
 
+### `internal/metadata/npm`
+
+The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks.
+
+### `internal/jsonscan`
+
+Reads and rewrites members of a JSON document in place, by byte offsets, without decoding the whole document. Shared by the npm and Composer metadata rewrites.
+
 ### `internal/gitcache`
 
 Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed.

internal/handler/composer.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/git-pkgs/proxy/internal/jsonscan"
 	"github.com/git-pkgs/proxy/internal/retention"
 	"github.com/git-pkgs/purl"
 )
@@ -219,7 +220,7 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str
 	if !json.Valid(body) {
 		return nil, nil, errors.New("composer metadata is not valid JSON")
 	}
-	format, _, err := lookupJSONString(body, "minified")
+	format, _, err := jsonscan.LookupString(body, "minified")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -229,17 +230,17 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str
 	var kept []string
 	listed := false
 	out.Grow(len(body) + len(body)/8)
-	err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) {
-		packages := m.value(body)
-		if !jsonKeyIs(m.key(body), "packages") || packages[0] != '{' {
+	err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) {
+		packages := m.Value(body)
+		if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' {
 			return false, nil
 		}
-		return true, rewriteJSONMembers(out, packages, func(out *bytes.Buffer, p jsonMember) (bool, error) {
-			versions := p.value(packages)
+		return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) {
+			versions := p.Value(packages)
 			if versions[0] != '[' {
 				return false, nil
 			}
-			packageName, err := jsonKey(p.key(packages))
+			packageName, err := jsonscan.Key(p.Key(packages))
 			if err != nil {
 				return false, err
 			}
@@ -279,8 +280,8 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v
 	_, routed := h.routes.match(packageName)
 
 	out.WriteByte('[')
-	err := forEachJSONElement(versions, func(entry []byte) error {
-		if minified && jsonStringIs(entry, composerDevReset) {
+	err := jsonscan.EachElement(versions, func(entry []byte) error {
+		if minified && jsonscan.StringIs(entry, composerDevReset) {
 			upstream.reset()
 			written.reset()
 			devReset = true
@@ -427,15 +428,15 @@ func disableDefaultNotification(fields *composerFields) {
 // rewriteDist returns a version's dist object with its url pointing at this
 // proxy. A dist without a string url is returned unchanged.
 func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte {
-	urlMember, ok, err := findJSONMember(dist, "url")
-	if err != nil || !ok || dist[urlMember.valStart] != '"' {
+	urlMember, ok, err := jsonscan.FindMember(dist, "url")
+	if err != nil || !ok || dist[urlMember.ValStart] != '"' {
 		return dist
 	}
 	var upstreamURL string
-	if err := json.Unmarshal(urlMember.value(dist), &upstreamURL); err != nil {
+	if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil {
 		return dist
 	}
-	distType, _, _ := lookupJSONString(dist, "type")
+	distType, _, _ := jsonscan.LookupString(dist, "type")
 
 	proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType)
 	if !ok {
@@ -445,10 +446,10 @@ func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte)
 	if err != nil {
 		return dist
 	}
-	rewritten := make([]byte, 0, len(dist)-len(urlMember.value(dist))+len(encoded))
-	rewritten = append(rewritten, dist[:urlMember.valStart]...)
+	rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded))
+	rewritten = append(rewritten, dist[:urlMember.ValStart]...)
 	rewritten = append(rewritten, encoded...)
-	return append(rewritten, dist[urlMember.valEnd:]...)
+	return append(rewritten, dist[urlMember.ValEnd:]...)
 }
 
 // proxyDistURL returns the URL this proxy serves a version's archive at, or
@@ -592,9 +593,9 @@ func (f *composerFields) set(name string, key, value []byte) {
 // apply overlays the members of a version entry. In minified metadata the
 // value "__unset" removes the field instead.
 func (f *composerFields) apply(entry []byte, minified bool) error {
-	return forEachJSONMember(entry, func(m jsonMember) error {
-		key, value := m.key(entry), m.value(entry)
-		if minified && jsonStringIs(value, composerUnset) {
+	return jsonscan.EachMember(entry, func(m jsonscan.Member) error {
+		key, value := m.Key(entry), m.Value(entry)
+		if minified && jsonscan.StringIs(value, composerUnset) {
 			value = nil
 		}
 		if bytes.IndexByte(key, '\\') < 0 {
@@ -603,7 +604,7 @@ func (f *composerFields) apply(entry []byte, minified bool) error {
 				return nil
 			}
 		}
-		name, err := jsonKey(key)
+		name, err := jsonscan.Key(key)
 		if err != nil {
 			return err
 		}
@@ -950,20 +951,20 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU
 // the minified format as it needs, since it runs on each archive the proxy
 // has not cached yet.
 func composerDistURL(body []byte, packageName, version string) (composerDist, error) {
-	format, _, err := lookupJSONString(body, "minified")
+	format, _, err := jsonscan.LookupString(body, "minified")
 	if err != nil {
 		return composerDist{}, err
 	}
-	versions, err := lookupJSON(body, "packages", packageName)
+	versions, err := jsonscan.Lookup(body, "packages", packageName)
 	if err != nil || len(versions) == 0 || versions[0] != '[' {
 		return composerDist{}, err
 	}
 	minified := format == composerMinified
 
 	fields := newComposerFields()
 	var dist composerDist
-	err = forEachJSONElement(versions, func(entry []byte) error {
-		if minified && jsonStringIs(entry, composerDevReset) {
+	err = jsonscan.EachElement(versions, func(entry []byte) error {
+		if minified && jsonscan.StringIs(entry, composerDevReset) {
 			fields.reset()
 			return nil
 		}
@@ -976,20 +977,20 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er
 		if err := fields.apply(entry, minified); err != nil {
 			return err
 		}
-		if !jsonStringIs(fields.get("version"), version) {
+		if !jsonscan.StringIs(fields.get("version"), version) {
 			return nil
 		}
 		distRaw := fields.get("dist")
-		dist.url, _, _ = lookupJSONString(distRaw, "url")
+		dist.url, _, _ = jsonscan.LookupString(distRaw, "url")
 		if dist.url != "" {
-			dist.distType, _, _ = lookupJSONString(distRaw, "type")
-			dist.shasum, _, _ = lookupJSONString(distRaw, "shasum")
+			dist.distType, _, _ = jsonscan.LookupString(distRaw, "type")
+			dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum")
 			dist.published = fields.get("time")
-			return errStopScan
+			return jsonscan.ErrStop
 		}
 		return nil
 	})
-	if errors.Is(err, errStopScan) {
+	if errors.Is(err, jsonscan.ErrStop) {
 		err = nil
 	}
 	return dist, err

internal/handler/denylist_test.go

@@ -80,15 +80,15 @@ func TestNPMDenylistMetadata(t *testing.T) {
 	h := NewNPMHandler(p, "https://proxy.example", "")
 	for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} {
 		body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}`
-		out, err := h.rewriteMetadata("@scope/demo", []byte(body))
+		out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(body))
 		if err != nil {
 			t.Fatal(err)
 		}
 		if strings.Contains(string(out), "2.0.0") || !strings.Contains(string(out), `"latest":"1.0.0"`) || !strings.Contains(string(out), `"next":"3.0.0-beta.1"`) {
 			t.Fatalf("incorrect metadata: %s", out)
 		}
 	}
-	out, err := h.rewriteMetadata("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`))
+	out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`))
 	if err != nil || strings.Contains(string(out), "2.0.0") {
 		t.Fatalf("all denied: %s, %v", out, err)
 	}
@@ -208,7 +208,7 @@ func TestDenylistAndCooldownCompose(t *testing.T) {
 	setTestDenylist(t, p, "pkg:npm/demo@1.0.0")
 	h := NewNPMHandler(p, "https://proxy.test", "")
 	body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339))
-	out, err := h.rewriteMetadata("demo", []byte(body))
+	out, _, err := h.rewriteMetadataKeeping("demo", []byte(body))
 	if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) {
 		t.Fatalf("combined policy: %s, %v", out, err)
 	}