diff --git a/docs/architecture.md b/docs/architecture.md index e7ce8c8f..660d8102 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,7 +34,7 @@ The proxy is a caching HTTP server that sits between package manager clients and 1. Client requests `GET /npm/lodash` 2. NPMHandler receives request 3. Handler fetches metadata from upstream `registry.npmjs.org/lodash` -4. Handler rewrites tarball URLs in metadata to point at proxy +4. Handler rewrites tarball URLs in metadata to point at proxy, and filters versions withheld by cooldown or the denylist (`internal/metadata/npm`) 5. Handler returns modified metadata to client Metadata is not cached - always fetched fresh. This ensures clients see new versions immediately. @@ -294,7 +294,7 @@ HTTP protocol handlers for each registry type. **NPMHandler:** - `handlePackageMetadata()` - Proxy + rewrite metadata - `handleDownload()` - Serve cached artifact -- Rewrites tarball URLs to point at proxy +- Fetches, caches and serves packuments, and caches their rewrites; the rewrite itself is `internal/metadata/npm` **CargoHandler:** - `handleConfig()` - Return registry config @@ -310,6 +310,14 @@ HTTP protocol handlers for each registry type. - Validates repository paths and the `repos` allowlist, and maps cache errors to statuses - Serves the Git LFS API under `{repo}.git/info/lfs/`: forwards download batches upstream and points each object at `objects/{oid}/{size}`, which is served from the artifact cache (ecosystem `git`, name `{name}/{repo}`, version the object ID); a miss uses the download URL kept from the batch or asks the upstream batch API for a fresh one, recorded without its signed query, copies the object from another repository when its content is already stored, and checks the object against its sha256 ID; a batch the upstream can't answer is answered from the cache +### `internal/metadata/npm` + +The npm metadata rewrite, separated from the handler so it can run where the proxy's infrastructure can't, such as WASI. It takes a packument's bytes and everything else as explicit inputs (proxy URL, cooldown policy, the time cooldown is evaluated at, denied versions, pushed versions) and returns the rewritten document with the versions it kept. It filters versions by cooldown and the denylist, moves `latest` off a withheld version, lists pushed versions in place of upstream ones, and points each tarball URL at the proxy, copying everything else byte for byte. It does no I/O and never reads the clock; callbacks report what it did, which the handler logs. It imports no database, storage, metrics, HTTP or logging packages, which a test checks. + +### `internal/jsonscan` + +Reads and rewrites members of a JSON document in place, by byte offsets, without decoding the whole document. Shared by the npm and Composer metadata rewrites. + ### `internal/gitcache` Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. diff --git a/internal/handler/composer.go b/internal/handler/composer.go index 7ced6d62..116d106e 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/jsonscan" "github.com/git-pkgs/proxy/internal/retention" "github.com/git-pkgs/purl" ) @@ -219,7 +220,7 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str if !json.Valid(body) { return nil, nil, errors.New("composer metadata is not valid JSON") } - format, _, err := lookupJSONString(body, "minified") + format, _, err := jsonscan.LookupString(body, "minified") if err != nil { return nil, nil, err } @@ -229,17 +230,17 @@ func (h *ComposerHandler) rewriteMetadataWithLocal(body []byte, localPackage str var kept []string listed := false out.Grow(len(body) + len(body)/8) - err = rewriteJSONMembers(&out, body, func(out *bytes.Buffer, m jsonMember) (bool, error) { - packages := m.value(body) - if !jsonKeyIs(m.key(body), "packages") || packages[0] != '{' { + err = jsonscan.RewriteMembers(&out, body, func(out *bytes.Buffer, m jsonscan.Member) (bool, error) { + packages := m.Value(body) + if !jsonscan.KeyIs(m.Key(body), "packages") || packages[0] != '{' { return false, nil } - return true, rewriteJSONMembers(out, packages, func(out *bytes.Buffer, p jsonMember) (bool, error) { - versions := p.value(packages) + return true, jsonscan.RewriteMembers(out, packages, func(out *bytes.Buffer, p jsonscan.Member) (bool, error) { + versions := p.Value(packages) if versions[0] != '[' { return false, nil } - packageName, err := jsonKey(p.key(packages)) + packageName, err := jsonscan.Key(p.Key(packages)) if err != nil { return false, err } @@ -279,8 +280,8 @@ func (h *ComposerHandler) writeVersions(out *bytes.Buffer, packageName string, v _, routed := h.routes.match(packageName) out.WriteByte('[') - err := forEachJSONElement(versions, func(entry []byte) error { - if minified && jsonStringIs(entry, composerDevReset) { + err := jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, composerDevReset) { upstream.reset() written.reset() devReset = true @@ -427,15 +428,15 @@ func disableDefaultNotification(fields *composerFields) { // rewriteDist returns a version's dist object with its url pointing at this // proxy. A dist without a string url is returned unchanged. func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) []byte { - urlMember, ok, err := findJSONMember(dist, "url") - if err != nil || !ok || dist[urlMember.valStart] != '"' { + urlMember, ok, err := jsonscan.FindMember(dist, "url") + if err != nil || !ok || dist[urlMember.ValStart] != '"' { return dist } var upstreamURL string - if err := json.Unmarshal(urlMember.value(dist), &upstreamURL); err != nil { + if err := json.Unmarshal(urlMember.Value(dist), &upstreamURL); err != nil { return dist } - distType, _, _ := lookupJSONString(dist, "type") + distType, _, _ := jsonscan.LookupString(dist, "type") proxyURL, ok := h.proxyDistURL(packageName, version, upstreamURL, distType) if !ok { @@ -445,10 +446,10 @@ func (h *ComposerHandler) rewriteDist(packageName, version string, dist []byte) if err != nil { return dist } - rewritten := make([]byte, 0, len(dist)-len(urlMember.value(dist))+len(encoded)) - rewritten = append(rewritten, dist[:urlMember.valStart]...) + rewritten := make([]byte, 0, len(dist)-len(urlMember.Value(dist))+len(encoded)) + rewritten = append(rewritten, dist[:urlMember.ValStart]...) rewritten = append(rewritten, encoded...) - return append(rewritten, dist[urlMember.valEnd:]...) + return append(rewritten, dist[urlMember.ValEnd:]...) } // proxyDistURL returns the URL this proxy serves a version's archive at, or @@ -592,9 +593,9 @@ func (f *composerFields) set(name string, key, value []byte) { // apply overlays the members of a version entry. In minified metadata the // value "__unset" removes the field instead. func (f *composerFields) apply(entry []byte, minified bool) error { - return forEachJSONMember(entry, func(m jsonMember) error { - key, value := m.key(entry), m.value(entry) - if minified && jsonStringIs(value, composerUnset) { + return jsonscan.EachMember(entry, func(m jsonscan.Member) error { + key, value := m.Key(entry), m.Value(entry) + if minified && jsonscan.StringIs(value, composerUnset) { value = nil } if bytes.IndexByte(key, '\\') < 0 { @@ -603,7 +604,7 @@ func (f *composerFields) apply(entry []byte, minified bool) error { return nil } } - name, err := jsonKey(key) + name, err := jsonscan.Key(key) if err != nil { return err } @@ -950,11 +951,11 @@ func (h *ComposerHandler) findDownloadURLFromMetadata(ctx context.Context, metaU // the minified format as it needs, since it runs on each archive the proxy // has not cached yet. func composerDistURL(body []byte, packageName, version string) (composerDist, error) { - format, _, err := lookupJSONString(body, "minified") + format, _, err := jsonscan.LookupString(body, "minified") if err != nil { return composerDist{}, err } - versions, err := lookupJSON(body, "packages", packageName) + versions, err := jsonscan.Lookup(body, "packages", packageName) if err != nil || len(versions) == 0 || versions[0] != '[' { return composerDist{}, err } @@ -962,8 +963,8 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er fields := newComposerFields() var dist composerDist - err = forEachJSONElement(versions, func(entry []byte) error { - if minified && jsonStringIs(entry, composerDevReset) { + err = jsonscan.EachElement(versions, func(entry []byte) error { + if minified && jsonscan.StringIs(entry, composerDevReset) { fields.reset() return nil } @@ -976,20 +977,20 @@ func composerDistURL(body []byte, packageName, version string) (composerDist, er if err := fields.apply(entry, minified); err != nil { return err } - if !jsonStringIs(fields.get("version"), version) { + if !jsonscan.StringIs(fields.get("version"), version) { return nil } distRaw := fields.get("dist") - dist.url, _, _ = lookupJSONString(distRaw, "url") + dist.url, _, _ = jsonscan.LookupString(distRaw, "url") if dist.url != "" { - dist.distType, _, _ = lookupJSONString(distRaw, "type") - dist.shasum, _, _ = lookupJSONString(distRaw, "shasum") + dist.distType, _, _ = jsonscan.LookupString(distRaw, "type") + dist.shasum, _, _ = jsonscan.LookupString(distRaw, "shasum") dist.published = fields.get("time") - return errStopScan + return jsonscan.ErrStop } return nil }) - if errors.Is(err, errStopScan) { + if errors.Is(err, jsonscan.ErrStop) { err = nil } return dist, err diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go index e89f186e..bb41e38a 100644 --- a/internal/handler/denylist_test.go +++ b/internal/handler/denylist_test.go @@ -80,7 +80,7 @@ func TestNPMDenylistMetadata(t *testing.T) { h := NewNPMHandler(p, "https://proxy.example", "") for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} { body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}` - out, err := h.rewriteMetadata("@scope/demo", []byte(body)) + out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(body)) if err != nil { t.Fatal(err) } @@ -88,7 +88,7 @@ func TestNPMDenylistMetadata(t *testing.T) { t.Fatalf("incorrect metadata: %s", out) } } - out, err := h.rewriteMetadata("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) + out, _, err := h.rewriteMetadataKeeping("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) if err != nil || strings.Contains(string(out), "2.0.0") { t.Fatalf("all denied: %s, %v", out, err) } @@ -208,7 +208,7 @@ func TestDenylistAndCooldownCompose(t *testing.T) { setTestDenylist(t, p, "pkg:npm/demo@1.0.0") h := NewNPMHandler(p, "https://proxy.test", "") body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339)) - out, err := h.rewriteMetadata("demo", []byte(body)) + out, _, err := h.rewriteMetadataKeeping("demo", []byte(body)) if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) { t.Fatalf("combined policy: %s, %v", out, err) } diff --git a/internal/handler/npm.go b/internal/handler/npm.go index b749a40a..c86dfd08 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -2,21 +2,18 @@ package handler import ( "bytes" - "encoding/json" "errors" "fmt" "io" - "maps" "net/http" "net/url" - "reflect" - "slices" - "sort" "strings" "sync" "time" "github.com/git-pkgs/integrity" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/metadata/npm" "github.com/git-pkgs/proxy/internal/retention" ) @@ -332,23 +329,12 @@ func (h *NPMHandler) packumentValidator() func([]byte) error { return requireFullPackument } -// requireFullPackument rejects an abbreviated packument: one with the -// top-level "modified" field the abbreviated format carries and no "time" -// map. A full document without a time map, as some registries serve, is -// let through as before, and so is anything that is not a JSON object, -// leaving it to the usual handling. +// requireFullPackument rejects an abbreviated packument (see +// npm.Abbreviated). A full document without a time map, as some registries +// serve, is let through as before, and so is anything that is not a JSON +// object, leaving it to the usual handling. func requireFullPackument(body []byte) error { - var hasTime, hasModified bool - err := forEachJSONMember(body, func(m jsonMember) error { - switch key := m.key(body); { - case jsonKeyIs(key, "time"): - hasTime = true - case jsonKeyIs(key, "modified"): - hasModified = true - } - return nil - }) - if err == nil && hasModified && !hasTime { + if npm.Abbreviated(body) { return errAbbreviatedPackument } return nil @@ -436,52 +422,13 @@ func (h *NPMHandler) npmVersionMetadataUsesFilteredPackument(packageName string) // rewriteNPMVersionFromPackument picks one version entry out of an already // filtered packument and rewrites its tarball URL. func (h *NPMHandler) rewriteNPMVersionFromPackument(packageName, specifier string, packument []byte) ([]byte, error) { - filtered, err := h.rewriteMetadata(packageName, packument) - if err != nil { - return nil, err - } - - target := specifier - if resolved, ok, err := lookupJSONString(filtered, "dist-tags", specifier); err != nil { - return nil, err - } else if ok { - target = resolved - } - - if h.proxy.versionDenied("npm", packageName, target) { - return nil, ErrVersionDenied - } - - versionRaw, err := lookupJSON(filtered, "versions", target) - if err != nil { - return nil, err - } - if len(versionRaw) == 0 { - return nil, ErrVersionNotFound - } - - var out bytes.Buffer - h.writeNPMVersion(&out, packageName, target, versionRaw) - return out.Bytes(), nil + out, err := npm.Version(packageName, specifier, packument, h.npmRewriteOptions(packageName, nil)) + return out, npmRewriteError(err) } -// rewriteMetadata rewrites tarball URLs in npm package metadata to point at this proxy. -// If cooldown is enabled, versions published too recently are filtered out. -// -// The document is never decoded as a whole. A full packument can run to tens -// of megabytes, and decoding it into generic maps took many times that in -// memory for every rewrite. Instead the version names, time map and dist-tags -// are decoded to decide what to keep, and the response is assembled from -// slices of the original bytes: only each version's tarball URL, and the time -// map and dist-tags when filtering changed them, are written fresh. Everything -// else reaches the client byte for byte as upstream sent it. -func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, error) { - out, _, err := h.rewriteMetadataKeeping(packageName, body) - return out, err -} - -// rewriteMetadataKeeping is rewriteMetadata that also reports the versions it -// kept. +// rewriteMetadataKeeping rewrites tarball URLs in npm package metadata to +// point at this proxy, filters the versions cooldown and the denylist +// withhold, and reports the versions it kept. See npm.Rewrite. func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([]byte, []string, error) { return h.rewriteMetadataWithLocal(packageName, body, nil) } @@ -491,394 +438,49 @@ func (h *NPMHandler) rewriteMetadataKeeping(packageName string, body []byte) ([] // version. Cooldown and the denylist only filter upstream versions, and // dist-tags stay as upstream has them. func (h *NPMHandler) rewriteMetadataWithLocal(packageName string, body []byte, local []npmLocalEntry) ([]byte, []string, error) { - doc, err := indexNPMPackument(body) - if err != nil { - return nil, nil, err - } - - versionsRaw := doc.value(doc.versionsAt) - if len(versionsRaw) == 0 || versionsRaw[0] != '{' { - version, ok, err := lookupJSONString(body, "version") - if err != nil { - return nil, nil, err - } - if ok { - if h.proxy.versionDenied("npm", packageName, version) { - return nil, nil, ErrVersionDenied - } - var out bytes.Buffer - if h.writeNPMVersion(&out, packageName, version, body) { - h.logTarballRewrites(packageName, 1) - } - return out.Bytes(), []string{version}, nil - } - if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { - return nil, nil, errors.New("npm metadata has no versions object") - } - return body, nil, nil // No versions to rewrite - } - - // The filters only look at which versions exist, so they run on a map of - // version names alongside the decoded time map and dist-tags. - versions := map[string]any{} - if err := forEachJSONMember(versionsRaw, func(m jsonMember) error { - version, err := jsonKey(m.key(versionsRaw)) - versions[version] = nil + out, kept, err := npm.Rewrite(packageName, body, h.npmRewriteOptions(packageName, local)) + return out, kept, npmRewriteError(err) +} + +// npmRewriteOptions gathers what a rewrite of packageName's metadata depends +// on from the proxy: its URL, the cooldown and denylist policy, the time +// cooldown is evaluated at, and the pushed versions to list. The callbacks +// log what the rewrite did. +func (h *NPMHandler) npmRewriteOptions(packageName string, local []npmLocalEntry) npm.Options { + return npm.Options{ + ProxyURL: h.proxyURL, + Cooldown: h.proxy.Cooldown, + DeniedVersions: h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)), + EvaluatedAt: time.Now(), + LocalVersions: local, + // Logged once per document: a line per version flooded debug logs, + // since packages such as eslint list hundreds of versions and with + // cooldown on every request rewrites the document again. + OnTarballsRewritten: func(count int) { + h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) + }, + OnCooldownFilter: func(version, publishedAt string) { + h.proxy.Logger.Info("cooldown: filtering npm version", "package", packageName, "version", version, + "published", publishedAt) + }, + } +} + +// npmRewriteError maps the rewrite's errors to the handler's, which the +// routes answer with 403 and 404. +func npmRewriteError(err error) error { + switch { + case errors.Is(err, npm.ErrVersionDenied): + return ErrVersionDenied + case errors.Is(err, npm.ErrVersionNotFound): + return ErrVersionNotFound + default: return err - }); err != nil { - return nil, nil, err } - metadata := map[string]any{} - timeMap := doc.decodeObject(doc.timeAt, metadata, "time") - distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") - timeEntries := len(timeMap) - originalTags := maps.Clone(distTags) - - h.applyCooldownFiltering(metadata, versions, packageName) - h.applyDenylistFiltering(metadata, versions, packageName) - for _, e := range local { - delete(versions, e.version) - } - - // Untouched members are copied as they are; the three the filters - // handled are written from their filtered state, and only when it changed. - out, err := doc.write(func(out *bytes.Buffer, i int) error { - switch { - case i == doc.versionsAt: - return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) - case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): - pushedAt := make([][2]string, 0, len(local)) - for _, e := range local { - pushedAt = append(pushedAt, [2]string{e.version, e.time}) - } - return writeFilteredJSONObjectWith(out, doc.value(i), func(key string) bool { - _, ok := timeMap[key] - _, pushed := findNPMLocalEntry(local, key) - return ok && !pushed - }, pushedAt) - case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): - encoded, err := json.Marshal(distTags) - out.Write(encoded) - return err - default: - out.Write(doc.value(i)) - return nil - } - }) - kept := slices.Collect(maps.Keys(versions)) - for _, e := range local { - kept = append(kept, e.version) - } - return out, kept, err -} - -// npmPackument is a packument's top-level members as offsets into its bytes, -// with the positions of the members rewriteMetadata filters, or -1 for those -// it lacks. A key that repeats is recorded at its last occurrence, the one -// JSON.parse keeps. -type npmPackument struct { - body []byte - members []jsonMember - versionsAt, timeAt, tagsAt int -} - -func indexNPMPackument(body []byte) (*npmPackument, error) { - if !json.Valid(body) { - return nil, errors.New("npm metadata is not valid JSON") - } - doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} - err := forEachJSONMember(body, func(m jsonMember) error { - switch key := m.key(body); { - case jsonKeyIs(key, "versions"): - doc.versionsAt = len(doc.members) - case jsonKeyIs(key, "time"): - doc.timeAt = len(doc.members) - case jsonKeyIs(key, "dist-tags"): - doc.tagsAt = len(doc.members) - } - doc.members = append(doc.members, m) - return nil - }) - return doc, err } -// value returns the raw value of member i, or nil when i is -1. -func (d *npmPackument) value(i int) []byte { - if i < 0 { - return nil - } - return d.members[i].value(d.body) -} - -// decodeObject decodes member i into metadata[name] and returns it when it is -// an object. The filters edit the returned map in place. -func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { - raw := d.value(i) - if raw == nil { - return nil - } - var value any - if err := json.Unmarshal(raw, &value); err != nil { - return nil - } - metadata[name] = value - object, _ := value.(map[string]any) - return object -} - -// write assembles the document in member order, with writeValue writing each -// member's value. Earlier duplicates of the filtered keys are dropped: they -// would carry unfiltered data, and JSON.parse ignores them anyway. -func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { - var out bytes.Buffer - out.Grow(len(d.body) + len(d.body)/8) - out.WriteByte('{') - first := true - for i, m := range d.members { - if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { - key := m.key(d.body) - if jsonKeyIs(key, "versions") || jsonKeyIs(key, "time") || jsonKeyIs(key, "dist-tags") { - continue - } - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.key(d.body)) - out.WriteByte(':') - if err := writeValue(&out, i); err != nil { - return nil, err - } - } - out.WriteByte('}') - return out.Bytes(), nil -} - -// writeNPMVersions writes the versions object, keeping the versions still in -// keep and pointing each kept version's tarball at this proxy, followed by the -// pushed versions in local. -func (h *NPMHandler) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []npmLocalEntry) error { - out.WriteByte('{') - first := true - rewritten := 0 - err := forEachJSONMember(versionsRaw, func(m jsonMember) error { - version, err := jsonKey(m.key(versionsRaw)) - if err != nil { - return err - } - if _, ok := keep[version]; !ok { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(m.key(versionsRaw)) - out.WriteByte(':') - if h.writeNPMVersion(out, packageName, version, m.value(versionsRaw)) { - rewritten++ - } - return nil - }) - h.logTarballRewrites(packageName, rewritten) - for _, e := range local { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(e.version) - out.Write(key) - out.WriteByte(':') - out.Write(e.entry) - } - out.WriteByte('}') - return err -} - -// writeNPMVersion writes one version entry with its dist.tarball pointing at -// this proxy, and reports whether it rewrote the tarball. An entry without a -// string tarball is written unchanged. -func (h *NPMHandler) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { - dist, ok, err := findJSONMember(raw, "dist") - if err != nil || !ok { - out.Write(raw) - return false - } - distRaw := dist.value(raw) - tarball, ok, err := findJSONMember(distRaw, "tarball") - if err != nil || !ok || distRaw[tarball.valStart] != '"' { - out.Write(raw) - return false - } - var oldTarball string - if err := json.Unmarshal(tarball.value(distRaw), &oldTarball); err != nil { - out.Write(raw) - return false - } - - newTarball := h.proxyTarballURL(packageName, version, oldTarball) - encoded, err := json.Marshal(newTarball) - if err != nil { - out.Write(raw) - return false - } - out.Write(raw[:dist.valStart+tarball.valStart]) - out.Write(encoded) - out.Write(raw[dist.valStart+tarball.valEnd:]) - return true -} - -// logTarballRewrites logs once per document how many tarball URLs it -// rewrote. A line per version flooded debug logs: packages such as eslint -// list hundreds of versions, and with cooldown on every request rewrites -// the document again. -func (h *NPMHandler) logTarballRewrites(packageName string, count int) { - if count > 0 { - h.proxy.Logger.Debug("rewrote tarball URLs", "package", packageName, "count", count) - } -} - -// writeFilteredJSONObjectWith keeps members whose decoded keys match keep, -// then appends extra string members. -func writeFilteredJSONObjectWith(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { - out.WriteByte('{') - first := true - err := forEachJSONMember(obj, func(m jsonMember) error { - key, err := jsonKey(m.key(obj)) - if err != nil { - return err - } - if !keep(key) { - return nil - } - if !first { - out.WriteByte(',') - } - first = false - out.Write(obj[m.keyStart:m.valEnd]) - return nil - }) - for _, member := range extra { - if !first { - out.WriteByte(',') - } - first = false - key, _ := json.Marshal(member[0]) - value, _ := json.Marshal(member[1]) - out.Write(key) - out.WriteByte(':') - out.Write(value) - } - out.WriteByte('}') - return err -} - -// applyCooldownFiltering removes versions that are too recently published, -// and updates dist-tags.latest if the current latest was filtered out. -func (h *NPMHandler) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return - } - - timeMap, _ := metadata["time"].(map[string]any) - if timeMap == nil { - return - } - - packagePURL := canonicalPackagePURL("npm", packageName) - - for version := range versions { - publishedStr, ok := timeMap[version].(string) - if !ok { - continue - } - publishedAt, err := time.Parse(time.RFC3339, publishedStr) - if err != nil { - continue - } - if !h.proxy.Cooldown.IsAllowed("npm", packagePURL, publishedAt) { - h.proxy.Logger.Info("cooldown: filtering npm version", - "package", packageName, "version", version, - "published", publishedStr) - delete(versions, version) - delete(timeMap, version) - } - } - - h.updateDistTagsLatest(metadata, versions, timeMap) -} - -// updateDistTagsLatest updates the dist-tags.latest field if the current latest -// version was removed by cooldown filtering. -func (h *NPMHandler) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { - distTags, ok := metadata["dist-tags"].(map[string]any) - if !ok { - return - } - - latest, ok := distTags["latest"].(string) - if !ok { - return - } - - if _, exists := versions[latest]; exists { - return - } - - if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { - distTags["latest"] = newLatest - } -} - -// proxyTarballURL returns the proxy URL that serves the tarball upstream -// lists at tarball for this version. func (h *NPMHandler) proxyTarballURL(packageName, version, tarball string) string { - filename := tarball - if idx := strings.LastIndex(tarball, "/"); idx >= 0 { - filename = tarball[idx+1:] - } - if h.extractVersionFromFilename(packageName, filename) != version { - _, shortName, scoped := strings.Cut(packageName, "/") - if !scoped { - shortName = packageName - } - filename = shortName + "-" + version + ".tgz" - } - - return fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, url.PathEscape(packageName), filename) -} - -// findNewestVersion returns the version string with the most recent timestamp -// from the remaining versions, using the time map. -func (h *NPMHandler) findNewestVersion(versions map[string]any, timeMap map[string]any) string { - if timeMap == nil { - return "" - } - - type versionTime struct { - version string - t time.Time - } - - var vts []versionTime - for v := range versions { - if ts, ok := timeMap[v].(string); ok { - if t, err := time.Parse(time.RFC3339, ts); err == nil { - vts = append(vts, versionTime{v, t}) - } - } - } - - if len(vts) == 0 { - return "" - } - - sort.Slice(vts, func(i, j int) bool { - return vts[i].t.After(vts[j].t) - }) - - return vts[0].version + return npm.TarballURL(h.proxyURL, packageName, version, tarball) } // handleDownload serves a package tarball, fetching and caching from upstream if needed. @@ -995,13 +597,13 @@ func (h *NPMHandler) getTarball(r *http.Request, registryURL, packageName, versi // returns "" when neither is usable, and the tarball is fetched unverified as // before. func npmVersionIntegrity(body []byte, version string) string { - sri, _, _ := lookupJSONString(body, "versions", version, "dist", "integrity") + sri, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "integrity") if sri != "" { if _, err := integrity.ParseSRI(sri); err == nil { return sri } } - shasum, _, _ := lookupJSONString(body, "versions", version, "dist", "shasum") + shasum, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "shasum") return sha1Checksum(shasum) } @@ -1009,7 +611,7 @@ func npmVersionIntegrity(body []byte, version string) string { // or "" if it has none. It reads that one value in place rather than decoding // every version, since it runs on each tarball the proxy has not cached yet. func npmVersionTarball(body []byte, version string) string { - tarball, _, _ := lookupJSONString(body, "versions", version, "dist", "tarball") + tarball, _, _ := jsonscan.LookupString(body, "versions", version, "dist", "tarball") return tarball } @@ -1077,7 +679,7 @@ func (h *NPMHandler) versionInCooldown(packageName, version string, metadata fun return false, err } - published, ok, err := lookupJSONString(body, "time", version) + published, ok, err := jsonscan.LookupString(body, "time", version) if err != nil { return false, fmt.Errorf("parsing npm metadata: %w", err) } @@ -1173,28 +775,6 @@ func (h *NPMHandler) parseDownloadPath(path string) (packageName, filename strin return packageName, filename } -// extractVersionFromFilename extracts version from npm tarball filename. -// e.g., "lodash-4.17.21.tgz" -> "4.17.21" -// e.g., "core-7.23.0.tgz" for @babel/core -> "7.23.0" func (h *NPMHandler) extractVersionFromFilename(packageName, filename string) string { - // Remove .tgz extension - if !strings.HasSuffix(filename, ".tgz") { - return "" - } - base := strings.TrimSuffix(filename, ".tgz") - - // For scoped packages, the filename uses the short name - shortName := packageName - if strings.Contains(packageName, "/") { - parts := strings.SplitN(packageName, "/", scopedParts) - shortName = parts[1] - } - - // Expected format: {shortName}-{version} - prefix := shortName + "-" - if !strings.HasPrefix(base, prefix) { - return "" - } - - return strings.TrimPrefix(base, prefix) + return npm.VersionFromFilename(packageName, filename) } diff --git a/internal/handler/npm_hosted.go b/internal/handler/npm_hosted.go index 4c786e8d..77f92277 100644 --- a/internal/handler/npm_hosted.go +++ b/internal/handler/npm_hosted.go @@ -8,15 +8,10 @@ import ( "time" "github.com/Masterminds/semver/v3" + "github.com/git-pkgs/proxy/internal/metadata/npm" ) -// npmLocalEntry is a pushed version as a packument lists it: the version -// entry built from its package.json, and the time it was pushed. -type npmLocalEntry struct { - version string - entry json.RawMessage - time string -} +type npmLocalEntry = npm.LocalVersion // npmLocalEntries returns the pushed versions of a package as packument // entries, or nil when there are none or hosted packages are disabled. @@ -42,14 +37,14 @@ func (h *NPMHandler) npmLocalEntries(packageName string) ([]npmLocalEntry, error if err != nil { return nil, err } - entries = append(entries, npmLocalEntry{version: lv.Version, entry: entry, time: lv.PushedAt.Format(time.RFC3339)}) + entries = append(entries, npmLocalEntry{Version: lv.Version, Entry: entry, PublishedAt: lv.PushedAt.Format(time.RFC3339)}) } return entries, nil } func findNPMLocalEntry(local []npmLocalEntry, version string) (npmLocalEntry, bool) { for _, e := range local { - if e.version == version { + if e.Version == version { return e, true } } @@ -81,7 +76,7 @@ func (h *NPMHandler) serveLocalMetadata(w http.ResponseWriter, r *http.Request, case req.versionLookup && pushed: w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.entry) + _, _ = w.Write(e.Entry) case h.proxy.hostedOnly("npm", req.packageName): h.serveHostedOnlyMetadata(w, r, req.packageName, req.specifier, req.versionLookup, req.metadataCacheKey, local) case len(local) > 0 && !req.versionLookup: @@ -145,7 +140,7 @@ func (h *NPMHandler) serveHostedOnlyMetadata(w http.ResponseWriter, r *http.Requ } w.Header().Set(headerContentType, contentTypeJSON) w.WriteHeader(http.StatusOK) - _, _ = w.Write(e.entry) + _, _ = w.Write(e.Entry) return } out, err := localNPMPackument(packageName, local) @@ -162,8 +157,8 @@ func localNPMPackument(packageName string, local []npmLocalEntry) ([]byte, error versions := make(map[string]json.RawMessage, len(local)) times := make(map[string]string, len(local)) for _, e := range local { - versions[e.version] = e.entry - times[e.version] = e.time + versions[e.Version] = e.Entry + times[e.Version] = e.PublishedAt } return json.Marshal(map[string]any{ "name": packageName, @@ -180,10 +175,10 @@ func latestNPMVersion(local []npmLocalEntry) string { var latestSemver *semver.Version semverOnly := true for _, e := range local { - if e.time > latestTime { - latest, latestTime = e.version, e.time + if e.PublishedAt > latestTime { + latest, latestTime = e.Version, e.PublishedAt } - v, err := semver.NewVersion(e.version) + v, err := semver.NewVersion(e.Version) if err != nil { semverOnly = false continue diff --git a/internal/handler/npm_portable_test.go b/internal/handler/npm_portable_test.go new file mode 100644 index 00000000..d0bb1931 --- /dev/null +++ b/internal/handler/npm_portable_test.go @@ -0,0 +1,107 @@ +package handler + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "strings" + "testing" + + "github.com/git-pkgs/cooldown" +) + +func TestNPMMetadataLogsPolicyAndURLChanges(t *testing.T) { + body, err := os.ReadFile("../metadata/npm/testdata/packument.json") + if err != nil { + t.Fatal(err) + } + f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") + var logs bytes.Buffer + f.proxy.Logger = slog.New(slog.NewJSONHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + if rec := f.get(t, "/@example/widgets"); rec.Code != http.StatusOK { + t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String()) + } + want := map[string]map[string]string{ + "cooldown: filtering npm version": {"level": "INFO", "package": "@example/widgets", "version": "3.0.0", "published": "2099-01-01T00:00:00Z"}, + "rewrote tarball URLs": {"level": "DEBUG", "package": "@example/widgets", "count": "1"}, + } + decoder := json.NewDecoder(&logs) + for { + var record map[string]any + if err := decoder.Decode(&record); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + message, _ := record["msg"].(string) + fields, ok := want[message] + if !ok { + continue + } + for key, value := range fields { + if fmt.Sprint(record[key]) != value { + t.Errorf("%s: %s = %v, want %q", message, key, record[key], value) + } + } + delete(want, message) + } + if len(want) != 0 { + t.Errorf("missing log records: %v", want) + } +} + +func TestNPMRoutesWithPortableMetadata(t *testing.T) { + body, err := os.ReadFile("../metadata/npm/testdata/packument.json") + if err != nil { + t.Fatal(err) + } + for _, hosted := range []bool{false, true} { + name := "upstream" + if hosted { + name = "hosted override" + } + t.Run(name, func(t *testing.T) { + f := newNPMHostedFixture(t, map[string]string{"/@example%2Fwidgets": string(body)}) + f.proxy.Cooldown = &cooldown.Config{Default: "7d"} + setTestDenylist(t, f.proxy, "pkg:npm/@example/widgets@2.0.0") + if hosted { + f.push(t, "@example/widgets", "3.0.0", `{"name":"@example/widgets","version":"3.0.0","dependencies":{"local":"1.0.0"}}`) + } + rec := f.get(t, "/@example/widgets") + doc := decodePackument(t, rec) + count := 1 + if hosted { + count = 2 + } + if len(doc.Versions) != count || doc.DistTags["latest"] != "1.0.0" || len(doc.DistTags) != 1 { + t.Fatalf("filtered metadata: %s", rec.Body.String()) + } + if doc.Versions["1.0.0"].Dist.Tarball != "http://proxy.local/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz" || !strings.Contains(rec.Body.String(), "12345678901234567890") { + t.Fatalf("rewritten metadata: %s", rec.Body.String()) + } + if hosted && doc.Versions["3.0.0"].Dependencies["local"] != "1.0.0" { + t.Fatalf("hosted override: %s", rec.Body.String()) + } + checkNPMVersionResponses(t, f, hosted) + }) + } +} + +func checkNPMVersionResponses(t *testing.T, f npmHostedFixture, hosted bool) { + t.Helper() + for specifier, want := range map[string]int{"latest": http.StatusOK, "1.0.0": http.StatusOK, "2.0.0": http.StatusForbidden, "3.0.0": http.StatusNotFound} { + if hosted && specifier == "3.0.0" { + want = http.StatusOK + } + got := f.get(t, "/@example/widgets/"+specifier) + if got.Code != want { + t.Errorf("%s: HTTP %d, want %d: %s", specifier, got.Code, want, got.Body.String()) + } + } +} diff --git a/internal/handler/npm_rewrite_bench_test.go b/internal/handler/npm_rewrite_bench_test.go index b200c5f8..86552db0 100644 --- a/internal/handler/npm_rewrite_bench_test.go +++ b/internal/handler/npm_rewrite_bench_test.go @@ -43,7 +43,7 @@ func BenchmarkNPMRewriteMetadata(b *testing.B) { b.SetBytes(int64(len(body))) b.ReportAllocs() for b.Loop() { - if _, err := h.rewriteMetadata("big", body); err != nil { + if _, _, err := h.rewriteMetadataKeeping("big", body); err != nil { b.Fatal(err) } } diff --git a/internal/handler/npm_rewrite_test.go b/internal/handler/npm_rewrite_test.go index 8521082c..1386503b 100644 --- a/internal/handler/npm_rewrite_test.go +++ b/internal/handler/npm_rewrite_test.go @@ -1,9 +1,7 @@ package handler import ( - "encoding/json" "errors" - "fmt" "net/http" "net/http/httptest" "strings" @@ -16,159 +14,6 @@ import ( "github.com/prometheus/client_golang/prometheus/testutil" ) -// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs -// change: everything else, including numbers encoding/json would round -// through float64 and characters it would HTML-escape, is copied verbatim. -func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + - `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` - - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) - if string(out) != want { - t.Errorf("rewrite =\n%s\nwant\n%s", out, want) - } -} - -func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{ - "versions": { - "1.0.0": "not an object", - "2.0.0": {"dist": "no object either"}, - "3.0.0": {"dist": {"tarball": 7}}, - "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, - "5.0.0": {"dist": {}} - } - }` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - var doc struct { - Versions map[string]json.RawMessage `json:"versions"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - for v, want := range map[string]string{ - "1.0.0": `"not an object"`, - "2.0.0": `{"dist": "no object either"}`, - "3.0.0": `{"dist": {"tarball": 7}}`, - "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, - "5.0.0": `{"dist": {}}`, - } { - if got := string(doc.Versions[v]); got != want { - t.Errorf("%s = %s, want %s", v, got, want) - } - } -} - -func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { - if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { - t.Errorf("%s: expected an error", input) - } - } -} - -func TestNPMRewriteMetadataNoVersions(t *testing.T) { - h := &NPMHandler{proxy: testProxy(), proxyURL: "http://proxy"} - input := `{"name":"demo","versions":[]}` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil || string(out) != input { - t.Errorf("rewrite = %s, %v; want input unchanged", out, err) - } - - setTestDenylist(t, h.proxy, "pkg:npm/demo@1.0.0") - if _, err := h.rewriteMetadata("demo", []byte(input)); err == nil { - t.Error("expected an error when a denylisted package has no versions object") - } -} - -// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and -// checks that versions, their time entries and the tags pointing at them are -// all removed, with latest moved to the newest version left. -func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { - now := time.Now() - ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } - - proxy := testProxy() - proxy.Cooldown = &cooldown.Config{Default: "3d"} - setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") - h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} - - version := func(v string) string { - return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) - } - input := `{"name":"demo",` + - `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + - `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + - `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` - - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - var doc struct { - DistTags map[string]string `json:"dist-tags"` - Versions map[string]json.RawMessage `json:"versions"` - Time map[string]string `json:"time"` - } - if err := json.Unmarshal(out, &doc); err != nil { - t.Fatalf("output is not JSON: %v\n%s", err, out) - } - if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { - t.Errorf("versions = %v, want only 1.0.0", keysOf(doc.Versions)) - } - if _, ok := doc.Time["2.0.0"]; ok { - t.Error("time still lists denylisted 2.0.0") - } - if _, ok := doc.Time["3.0.0"]; ok { - t.Error("time still lists 3.0.0, which is inside cooldown") - } - if _, ok := doc.Time["created"]; !ok { - t.Error("time lost its created entry") - } - want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} - if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { - t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) - } - if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { - t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) - } -} - -// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key -// cannot smuggle an unfiltered copy of the versions past the filters. -func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { - proxy := testProxy() - setTestDenylist(t, proxy, "pkg:npm/demo@2.0.0") - h := &NPMHandler{proxy: proxy, proxyURL: "http://proxy"} - input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + - `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` - out, err := h.rewriteMetadata("demo", []byte(input)) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { - t.Errorf("rewrite = %s", out) - } -} - -func keysOf(m map[string]json.RawMessage) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return keys -} - // TestMetadataServesRewriteWithoutReadingStorage checks that a repeat // request inside the metadata TTL is answered from the rewrite cache by the // stored digest alone, without reading the cached document back. diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index 08677767..0e083a3d 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -155,7 +155,7 @@ func TestNPMRewriteMetadata(t *testing.T) { } }` - output, err := h.rewriteMetadata("lodash", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -186,7 +186,7 @@ func TestNPMRewriteMetadataLogsOncePerDocument(t *testing.T) { "4.17.19":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz"}}, "4.17.20":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz"}}, "4.17.21":{"dist":{"tarball":"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"}}}}` - if _, err := h.rewriteMetadata("lodash", []byte(input)); err != nil { + if _, _, err := h.rewriteMetadataKeeping("lodash", []byte(input)); err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -215,7 +215,7 @@ func TestNPMRewriteMetadataScopedPackage(t *testing.T) { } }` - output, err := h.rewriteMetadata("@babel/core", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("@babel/core", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -330,7 +330,7 @@ func TestNPMRewriteMetadataCooldown(t *testing.T) { } }` - output, err := h.rewriteMetadata("testpkg", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } @@ -386,7 +386,7 @@ func TestNPMRewriteMetadataCooldownExemptPackage(t *testing.T) { } }` - output, err := h.rewriteMetadata("testpkg", []byte(input)) + output, _, err := h.rewriteMetadataKeeping("testpkg", []byte(input)) if err != nil { t.Fatalf("rewriteMetadata failed: %v", err) } diff --git a/internal/handler/jsonscan.go b/internal/jsonscan/jsonscan.go similarity index 52% rename from internal/handler/jsonscan.go rename to internal/jsonscan/jsonscan.go index 2c95aa43..7fcade77 100644 --- a/internal/handler/jsonscan.go +++ b/internal/jsonscan/jsonscan.go @@ -1,39 +1,39 @@ -package handler - -import ( - "bytes" - "encoding/json" - "errors" -) - -// The helpers in this file walk a JSON document in place, reporting where -// object members sit as offsets into the original bytes. They let a handler +// Package jsonscan walks a JSON document in place, reporting where +// object members sit as offsets into the original bytes. It lets a caller // read or replace one value in a large metadata document without decoding the // rest of it into Go values, which for an npm packument costs many times the // document's size. // -// They check structure only as far as they need to find their way through +// It checks structure only as far as it needs to find its way through // the document. Callers that copy unvisited bytes into a response validate the // whole document first with json.Valid. +package jsonscan + +import ( + "bytes" + "encoding/json" + "errors" +) -var errMalformedJSON = errors.New("malformed JSON") +// ErrMalformed is returned when the document is not well-formed JSON. +var ErrMalformed = errors.New("malformed JSON") -// errNotJSONObject is returned when a value expected to be an object is not. -var errNotJSONObject = errors.New("JSON value is not an object") +// ErrNotObject is returned when a value expected to be an object is not. +var ErrNotObject = errors.New("JSON value is not an object") -// errStopScan is returned by a callback to end a walk early once it has +// ErrStop is returned by a callback to end a walk early once it has // found what it was looking for. -var errStopScan = errors.New("stop scanning") +var ErrStop = errors.New("stop scanning") -// jsonMember is one member of a JSON object as offsets into the bytes that +// Member is one member of a JSON object as offsets into the bytes that // were scanned. The key span includes its quotes. -type jsonMember struct { - keyStart, keyEnd int - valStart, valEnd int +type Member struct { + KeyStart, KeyEnd int + ValStart, ValEnd int } -func (m jsonMember) key(data []byte) []byte { return data[m.keyStart:m.keyEnd] } -func (m jsonMember) value(data []byte) []byte { return data[m.valStart:m.valEnd] } +func (m Member) Key(data []byte) []byte { return data[m.KeyStart:m.KeyEnd] } +func (m Member) Value(data []byte) []byte { return data[m.ValStart:m.ValEnd] } func skipJSONSpace(data []byte, i int) int { for i < len(data) { @@ -57,13 +57,13 @@ func skipJSONString(data []byte, i int) (int, error) { return j + 1, nil } } - return 0, errMalformedJSON + return 0, ErrMalformed } // skipJSONValue returns the index just past the value starting at data[i]. func skipJSONValue(data []byte, i int) (int, error) { if i >= len(data) { - return 0, errMalformedJSON + return 0, ErrMalformed } switch data[i] { case '"': @@ -87,9 +87,9 @@ func skipJSONValue(data []byte, i int) (int, error) { } } } - return 0, errMalformedJSON + return 0, ErrMalformed case '}', ']', ',', ':': - return 0, errMalformedJSON + return 0, ErrMalformed default: // A number, true, false or null runs to the next delimiter. j := i @@ -104,13 +104,13 @@ func skipJSONValue(data []byte, i int) (int, error) { } } -// forEachJSONMember calls fn for each member of the object obj holds, in -// document order. It returns errNotJSONObject if obj is not an object, and +// EachMember calls fn for each member of the object obj holds, in +// document order. It returns ErrNotObject if obj is not an object, and // stops early with fn's error if fn returns one. -func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { +func EachMember(obj []byte, fn func(Member) error) error { i := skipJSONSpace(obj, 0) if i >= len(obj) || obj[i] != '{' { - return errNotJSONObject + return ErrNotObject } i = skipJSONSpace(obj, i+1) if i < len(obj) && obj[i] == '}' { @@ -118,7 +118,7 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { } for { if i >= len(obj) || obj[i] != '"' { - return errMalformedJSON + return ErrMalformed } keyEnd, err := skipJSONString(obj, i) if err != nil { @@ -126,19 +126,19 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { } colon := skipJSONSpace(obj, keyEnd) if colon >= len(obj) || obj[colon] != ':' { - return errMalformedJSON + return ErrMalformed } valStart := skipJSONSpace(obj, colon+1) valEnd, err := skipJSONValue(obj, valStart) if err != nil { return err } - if err := fn(jsonMember{keyStart: i, keyEnd: keyEnd, valStart: valStart, valEnd: valEnd}); err != nil { + if err := fn(Member{KeyStart: i, KeyEnd: keyEnd, ValStart: valStart, ValEnd: valEnd}); err != nil { return err } next := skipJSONSpace(obj, valEnd) if next >= len(obj) { - return errMalformedJSON + return ErrMalformed } switch obj[next] { case ',': @@ -146,17 +146,17 @@ func forEachJSONMember(obj []byte, fn func(jsonMember) error) error { case '}': return nil default: - return errMalformedJSON + return ErrMalformed } } } -// forEachJSONElement calls fn with each element of the array arr holds, in +// EachElement calls fn with each element of the array arr holds, in // order, and stops early with fn's error if fn returns one. -func forEachJSONElement(arr []byte, fn func(element []byte) error) error { +func EachElement(arr []byte, fn func(element []byte) error) error { i := skipJSONSpace(arr, 0) if i >= len(arr) || arr[i] != '[' { - return errMalformedJSON + return ErrMalformed } i = skipJSONSpace(arr, i+1) if i < len(arr) && arr[i] == ']' { @@ -172,7 +172,7 @@ func forEachJSONElement(arr []byte, fn func(element []byte) error) error { } next := skipJSONSpace(arr, end) if next >= len(arr) { - return errMalformedJSON + return ErrMalformed } switch arr[next] { case ',': @@ -180,22 +180,22 @@ func forEachJSONElement(arr []byte, fn func(element []byte) error) error { case ']': return nil default: - return errMalformedJSON + return ErrMalformed } } } -// rewriteJSONMembers copies the object obj holds to out byte for byte, +// RewriteMembers copies the object obj holds to out byte for byte, // except that rewrite may write a member's value itself. It returns true // when it did; otherwise the original value is copied. -func rewriteJSONMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m jsonMember) (bool, error)) error { +func RewriteMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.Buffer, m Member) (bool, error)) error { copied := 0 - err := forEachJSONMember(obj, func(m jsonMember) error { - out.Write(obj[copied:m.valStart]) - copied = m.valStart + err := EachMember(obj, func(m Member) error { + out.Write(obj[copied:m.ValStart]) + copied = m.ValStart rewritten, err := rewrite(out, m) if rewritten { - copied = m.valEnd + copied = m.ValEnd } return err }) @@ -203,8 +203,8 @@ func rewriteJSONMembers(out *bytes.Buffer, obj []byte, rewrite func(out *bytes.B return err } -// jsonKey decodes a quoted key as forEachJSONMember reports it. -func jsonKey(raw []byte) (string, error) { +// Key decodes a quoted key as EachMember reports it. +func Key(raw []byte) (string, error) { if bytes.IndexByte(raw, '\\') < 0 { return string(raw[1 : len(raw)-1]), nil } @@ -213,27 +213,27 @@ func jsonKey(raw []byte) (string, error) { return key, err } -// jsonKeyIs reports whether a quoted key decodes to name. -func jsonKeyIs(raw []byte, name string) bool { +// KeyIs reports whether a quoted key decodes to name. +func KeyIs(raw []byte, name string) bool { if bytes.IndexByte(raw, '\\') < 0 { return string(raw[1:len(raw)-1]) == name } - key, err := jsonKey(raw) + key, err := Key(raw) return err == nil && key == name } -// jsonStringIs reports whether raw is a JSON string that decodes to s. -func jsonStringIs(raw []byte, s string) bool { - return len(raw) >= 2 && raw[0] == '"' && jsonKeyIs(raw, s) +// StringIs reports whether raw is a JSON string that decodes to s. +func StringIs(raw []byte, s string) bool { + return len(raw) >= 2 && raw[0] == '"' && KeyIs(raw, s) } -// findJSONMember returns the member of obj named name. When the key repeats, +// FindMember returns the member of obj named name. When the key repeats, // the last one wins, as it does for JSON.parse and encoding/json. -func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { - var found jsonMember +func FindMember(obj []byte, name string) (Member, bool, error) { + var found Member ok := false - err := forEachJSONMember(obj, func(m jsonMember) error { - if jsonKeyIs(m.key(obj), name) { + err := EachMember(obj, func(m Member) error { + if KeyIs(m.Key(obj), name) { found, ok = m, true } return nil @@ -241,28 +241,28 @@ func findJSONMember(obj []byte, name string) (jsonMember, bool, error) { return found, ok, err } -// lookupJSON follows path through nested objects in doc and returns the +// Lookup follows path through nested objects in doc and returns the // value at its end. It returns nil and no error when a key along the path is // missing or names something other than an object. -func lookupJSON(doc []byte, path ...string) ([]byte, error) { +func Lookup(doc []byte, path ...string) ([]byte, error) { value := doc for _, name := range path { - m, ok, err := findJSONMember(value, name) - if errors.Is(err, errNotJSONObject) { + m, ok, err := FindMember(value, name) + if errors.Is(err, ErrNotObject) { return nil, nil } if err != nil || !ok { return nil, err } - value = m.value(value) + value = m.Value(value) } return value, nil } -// lookupJSONString is lookupJSON for a string value. ok is false when the +// LookupString is Lookup for a string value. ok is false when the // value is missing or is not a string. -func lookupJSONString(doc []byte, path ...string) (string, bool, error) { - raw, err := lookupJSON(doc, path...) +func LookupString(doc []byte, path ...string) (string, bool, error) { + raw, err := Lookup(doc, path...) if err != nil || len(raw) == 0 || raw[0] != '"' { return "", false, err } @@ -272,3 +272,38 @@ func lookupJSONString(doc []byte, path ...string) (string, bool, error) { } return s, true, nil } + +// WriteFilteredObject writes the object obj holds, keeping the members +// whose decoded key keep accepts, followed by extra string members. +func WriteFilteredObject(out *bytes.Buffer, obj []byte, keep func(string) bool, extra [][2]string) error { + out.WriteByte('{') + first := true + err := EachMember(obj, func(m Member) error { + key, err := Key(m.Key(obj)) + if err != nil { + return err + } + if !keep(key) { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(obj[m.KeyStart:m.ValEnd]) + return nil + }) + for _, member := range extra { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(member[0]) + value, _ := json.Marshal(member[1]) + out.Write(key) + out.WriteByte(':') + out.Write(value) + } + out.WriteByte('}') + return err +} diff --git a/internal/handler/jsonscan_test.go b/internal/jsonscan/jsonscan_test.go similarity index 71% rename from internal/handler/jsonscan_test.go rename to internal/jsonscan/jsonscan_test.go index 0825000b..a14ac855 100644 --- a/internal/handler/jsonscan_test.go +++ b/internal/jsonscan/jsonscan_test.go @@ -1,4 +1,4 @@ -package handler +package jsonscan import ( "bytes" @@ -34,7 +34,7 @@ func TestLookupJSONString(t *testing.T) { {[]string{"name", "x"}, "", false}, } for _, c := range cases { - got, ok, err := lookupJSONString(doc, c.path...) + got, ok, err := LookupString(doc, c.path...) if err != nil || ok != c.wantOK || got != c.want { t.Errorf("lookup %v = %q, %v, %v; want %q, %v", c.path, got, ok, err, c.want, c.wantOK) } @@ -43,7 +43,7 @@ func TestLookupJSONString(t *testing.T) { func TestLookupJSONEscapedKey(t *testing.T) { doc := []byte(`{"versions": {"1.0.0": {"dist": {"tarball": "u"}}}}`) - got, ok, err := lookupJSONString(doc, "versions", "1.0.0", "dist", "tarball") + got, ok, err := LookupString(doc, "versions", "1.0.0", "dist", "tarball") if err != nil || !ok || got != "u" { t.Errorf("lookup = %q, %v, %v", got, ok, err) } @@ -57,9 +57,9 @@ func TestLookupJSONMalformed(t *testing.T) { `{"a": "unterminated}`, `{"a": [1, 2}`, } { - _, _, err := lookupJSONString([]byte(doc), "versions", "1.0.0", "dist", "tarball") - if !errors.Is(err, errMalformedJSON) { - t.Errorf("%s: err = %v, want errMalformedJSON", doc, err) + _, _, err := LookupString([]byte(doc), "versions", "1.0.0", "dist", "tarball") + if !errors.Is(err, ErrMalformed) { + t.Errorf("%s: err = %v, want ErrMalformed", doc, err) } } } @@ -67,8 +67,8 @@ func TestLookupJSONMalformed(t *testing.T) { func TestForEachJSONMemberOffsets(t *testing.T) { doc := []byte(` { "a" : 1 , "b":{"c":[true,null]} ,"d":"x\"y" } `) var got [][2]string - err := forEachJSONMember(doc, func(m jsonMember) error { - got = append(got, [2]string{string(m.key(doc)), string(m.value(doc))}) + err := EachMember(doc, func(m Member) error { + got = append(got, [2]string{string(m.Key(doc)), string(m.Value(doc))}) return nil }) want := [][2]string{{`"a"`, `1`}, {`"b"`, `{"c":[true,null]}`}, {`"d"`, `"x\"y"`}} @@ -81,17 +81,17 @@ func TestForEachJSONMemberOffsets(t *testing.T) { } } - if err := forEachJSONMember([]byte(`[1]`), func(jsonMember) error { return nil }); !errors.Is(err, errNotJSONObject) { - t.Errorf("array: err = %v, want errNotJSONObject", err) + if err := EachMember([]byte(`[1]`), func(Member) error { return nil }); !errors.Is(err, ErrNotObject) { + t.Errorf("array: err = %v, want ErrNotObject", err) } - if err := forEachJSONMember([]byte(`{}`), func(jsonMember) error { t.Error("called for empty object"); return nil }); err != nil { + if err := EachMember([]byte(`{}`), func(Member) error { t.Error("called for empty object"); return nil }); err != nil { t.Errorf("empty object: err = %v", err) } } func TestWriteFilteredJSONObject(t *testing.T) { var out bytes.Buffer - err := writeFilteredJSONObjectWith(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) + err := WriteFilteredObject(&out, []byte(`{"a": 1, "b": {"x": 2}, "c": 3}`), func(k string) bool { return k != "b" }, nil) if err != nil || out.String() != `{"a": 1,"c": 3}` { t.Errorf("filtered = %s, %v", out.String(), err) } @@ -99,7 +99,7 @@ func TestWriteFilteredJSONObject(t *testing.T) { func TestForEachJSONElement(t *testing.T) { var got []string - err := forEachJSONElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { + err := EachElement([]byte(` [ 1, "a,]", {"b": [2]} ,[] ] `), func(e []byte) error { got = append(got, string(e)) return nil }) @@ -113,11 +113,11 @@ func TestForEachJSONElement(t *testing.T) { } } - if err := forEachJSONElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { + if err := EachElement([]byte(`[]`), func([]byte) error { t.Error("called for empty array"); return nil }); err != nil { t.Errorf("empty array: err = %v", err) } for _, bad := range []string{`{}`, `[1,`, `[1 2]`, `[1`} { - if err := forEachJSONElement([]byte(bad), func([]byte) error { return nil }); err == nil { + if err := EachElement([]byte(bad), func([]byte) error { return nil }); err == nil { t.Errorf("%s: expected an error", bad) } } @@ -126,8 +126,8 @@ func TestForEachJSONElement(t *testing.T) { func TestRewriteJSONMembers(t *testing.T) { obj := []byte(`{ "a": 1, "b": [2], "c" : 3 }`) var out bytes.Buffer - err := rewriteJSONMembers(&out, obj, func(out *bytes.Buffer, m jsonMember) (bool, error) { - if !jsonKeyIs(m.key(obj), "b") { + err := RewriteMembers(&out, obj, func(out *bytes.Buffer, m Member) (bool, error) { + if !KeyIs(m.Key(obj), "b") { return false, nil } out.WriteString(`"B"`) @@ -148,8 +148,8 @@ func TestJSONStringIs(t *testing.T) { ``: false, `["~dev"]`: false, } { - if got := jsonStringIs([]byte(raw), "~dev"); got != want { - t.Errorf("jsonStringIs(%s) = %v, want %v", raw, got, want) + if got := StringIs([]byte(raw), "~dev"); got != want { + t.Errorf("StringIs(%s) = %v, want %v", raw, got, want) } } } diff --git a/internal/handler/npm_denylist.go b/internal/metadata/npm/denylist.go similarity index 82% rename from internal/handler/npm_denylist.go rename to internal/metadata/npm/denylist.go index 7f8b87db..d34d4d34 100644 --- a/internal/handler/npm_denylist.go +++ b/internal/metadata/npm/denylist.go @@ -1,12 +1,9 @@ -package handler +package npm import "github.com/Masterminds/semver/v3" -func (h *NPMHandler) applyDenylistFiltering(metadata, versions map[string]any, packageName string) { - if h.proxy.Denylist == nil { - return - } - denied := h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)) +func (h *transformer) applyDenylistFiltering(metadata, versions map[string]any) { + denied := h.DeniedVersions if len(denied) == 0 { return } diff --git a/internal/metadata/npm/imports_test.go b/internal/metadata/npm/imports_test.go new file mode 100644 index 00000000..0f04bc18 --- /dev/null +++ b/internal/metadata/npm/imports_test.go @@ -0,0 +1,47 @@ +package npm + +import ( + "os/exec" + "slices" + "strings" + "testing" +) + +// TestPortableImports keeps the package free of the proxy's infrastructure: +// no handler, database, storage, metrics, HTTP or logging, so it builds for +// targets such as WASI where those are not available. +func TestPortableImports(t *testing.T) { + goTool, err := exec.LookPath("go") + if err != nil { + t.Skip("go tool not found") + } + out, err := exec.Command(goTool, "list", "-deps", "-f", "{{.Standard}} {{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + allowed := []string{ + "github.com/Masterminds/semver/v3", + "github.com/git-pkgs/cooldown", + "github.com/git-pkgs/proxy/internal/jsonscan", + "github.com/git-pkgs/proxy/internal/metadata/npm", + "github.com/git-pkgs/proxy/internal/packageurl", + "github.com/git-pkgs/purl", + "github.com/git-pkgs/vers", + "github.com/package-url/packageurl-go", + } + // database/sql/driver comes in through semver, which implements its + // Valuer interface; it is only interfaces, no driver. + forbiddenStd := []string{"database/sql", "log/slog", "net", "net/http"} + for line := range strings.Lines(string(out)) { + standard, path, _ := strings.Cut(strings.TrimSpace(line), " ") + if standard == "true" { + if slices.Contains(forbiddenStd, path) { + t.Errorf("imports %s", path) + } + continue + } + if !slices.Contains(allowed, path) { + t.Errorf("imports %s, which is not on the portable list", path) + } + } +} diff --git a/internal/metadata/npm/metadata.go b/internal/metadata/npm/metadata.go new file mode 100644 index 00000000..62e531f0 --- /dev/null +++ b/internal/metadata/npm/metadata.go @@ -0,0 +1,490 @@ +// Package npm transforms registry metadata without storage or network access. +package npm + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + "net/url" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/jsonscan" + "github.com/git-pkgs/proxy/internal/packageurl" +) + +var ErrVersionDenied = errors.New("package version blocked by denylist") +var ErrVersionNotFound = errors.New("version not found") + +// CooldownPolicy evaluates version age at the caller's supplied time. +type CooldownPolicy interface { + Enabled() bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision +} + +// LocalVersion is an already approved hosted version and its publication time. +type LocalVersion struct { + Version string + Entry json.RawMessage + PublishedAt string +} + +// Options supplies the public URL, policy and hosted versions for a rewrite. +// EvaluatedAt is used as given; the transformation never reads the clock. +type Options struct { + ProxyURL string + Cooldown CooldownPolicy + DeniedVersions map[string]bool + EvaluatedAt time.Time + LocalVersions []LocalVersion + // Optional callbacks report changes synchronously without retaining events. + // OnTarballsRewritten is called at most once per document, with how many + // tarball URLs it pointed at the proxy, and not at all when none. + OnTarballsRewritten func(count int) + OnCooldownFilter func(version, publishedAt string) +} + +type transformer struct{ Options } + +// Rewrite filters upstream versions and rewrites their tarball URLs, preserving +// untouched JSON values verbatim. Hosted entries override upstream versions. +// The returned version set is unordered. The document, denylist and hosted +// entries are not modified. +func Rewrite(packageName string, body []byte, options Options) ([]byte, []string, error) { + return (&transformer{options}).rewriteMetadataWithLocal(packageName, body, options.LocalVersions) +} + +// Version resolves a version or dist-tag through the filtered packument. +func Version(packageName, specifier string, packument []byte, options Options) ([]byte, error) { + filtered, _, err := Rewrite(packageName, packument, options) + if err != nil { + return nil, err + } + + target := specifier + if resolved, ok, err := jsonscan.LookupString(filtered, "dist-tags", specifier); err != nil { + return nil, err + } else if ok { + target = resolved + } + + if options.DeniedVersions[target] { + return nil, ErrVersionDenied + } + + versionRaw, err := jsonscan.Lookup(filtered, "versions", target) + if err != nil { + return nil, err + } + if len(versionRaw) == 0 { + return nil, ErrVersionNotFound + } + + var out bytes.Buffer + (&transformer{options}).writeNPMVersion(&out, packageName, target, versionRaw) + return out.Bytes(), nil +} + +// Abbreviated reports whether body is an abbreviated packument: one with the +// top-level "modified" field the abbreviated format carries and no "time" +// map, which cooldown needs. A full document without a time map, as some +// registries serve, is not abbreviated, and neither is anything that is not +// a JSON object. +func Abbreviated(body []byte) bool { + var hasTime, hasModified bool + err := jsonscan.EachMember(body, func(m jsonscan.Member) error { + switch key := m.Key(body); { + case jsonscan.KeyIs(key, "time"): + hasTime = true + case jsonscan.KeyIs(key, "modified"): + hasModified = true + } + return nil + }) + return err == nil && hasModified && !hasTime +} + +func (h *transformer) rewriteMetadataWithLocal(packageName string, body []byte, local []LocalVersion) ([]byte, []string, error) { + doc, err := indexNPMPackument(body) + if err != nil { + return nil, nil, err + } + + versionsRaw := doc.value(doc.versionsAt) + if len(versionsRaw) == 0 || versionsRaw[0] != '{' { + version, ok, err := jsonscan.LookupString(body, "version") + if err != nil { + return nil, nil, err + } + if ok { + if h.DeniedVersions[version] { + return nil, nil, ErrVersionDenied + } + var out bytes.Buffer + if h.writeNPMVersion(&out, packageName, version, body) { + h.tarballsRewritten(1) + } + return out.Bytes(), []string{version}, nil + } + if len(h.DeniedVersions) != 0 { + return nil, nil, errors.New("npm metadata has no versions object") + } + return body, nil, nil // No versions to rewrite + } + + // The filters only look at which versions exist, so they run on a map of + // version names alongside the decoded time map and dist-tags. + versions := map[string]any{} + if err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { + version, err := jsonscan.Key(m.Key(versionsRaw)) + versions[version] = nil + return err + }); err != nil { + return nil, nil, err + } + metadata := map[string]any{} + timeMap := doc.decodeObject(doc.timeAt, metadata, "time") + distTags := doc.decodeObject(doc.tagsAt, metadata, "dist-tags") + timeEntries := len(timeMap) + originalTags := maps.Clone(distTags) + + h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions) + for _, e := range local { + delete(versions, e.Version) + } + + // Untouched members are copied as they are; the three the filters + // handled are written from their filtered state, and only when it changed. + out, err := doc.write(func(out *bytes.Buffer, i int) error { + switch { + case i == doc.versionsAt: + return h.writeNPMVersions(out, packageName, versionsRaw, versions, local) + case i == doc.timeAt && timeMap != nil && (len(timeMap) != timeEntries || len(local) > 0): + pushedAt := make([][2]string, 0, len(local)) + for _, e := range local { + pushedAt = append(pushedAt, [2]string{e.Version, e.PublishedAt}) + } + return jsonscan.WriteFilteredObject(out, doc.value(i), func(key string) bool { + _, ok := timeMap[key] + pushed := slices.ContainsFunc(local, func(e LocalVersion) bool { return e.Version == key }) + return ok && !pushed + }, pushedAt) + case i == doc.tagsAt && distTags != nil && !reflect.DeepEqual(distTags, originalTags): + encoded, err := json.Marshal(distTags) + out.Write(encoded) + return err + default: + out.Write(doc.value(i)) + return nil + } + }) + kept := slices.Collect(maps.Keys(versions)) + for _, e := range local { + kept = append(kept, e.Version) + } + return out, kept, err +} + +// npmPackument is a packument's top-level members as offsets into its bytes, +// with the positions of the members rewriteMetadata filters, or -1 for those +// it lacks. A key that repeats is recorded at its last occurrence, the one +// JSON.parse keeps. +type npmPackument struct { + body []byte + members []jsonscan.Member + versionsAt, timeAt, tagsAt int +} + +func indexNPMPackument(body []byte) (*npmPackument, error) { + if !json.Valid(body) { + return nil, errors.New("npm metadata is not valid JSON") + } + doc := &npmPackument{body: body, versionsAt: -1, timeAt: -1, tagsAt: -1} + err := jsonscan.EachMember(body, func(m jsonscan.Member) error { + switch key := m.Key(body); { + case jsonscan.KeyIs(key, "versions"): + doc.versionsAt = len(doc.members) + case jsonscan.KeyIs(key, "time"): + doc.timeAt = len(doc.members) + case jsonscan.KeyIs(key, "dist-tags"): + doc.tagsAt = len(doc.members) + } + doc.members = append(doc.members, m) + return nil + }) + return doc, err +} + +// value returns the raw value of member i, or nil when i is -1. +func (d *npmPackument) value(i int) []byte { + if i < 0 { + return nil + } + return d.members[i].Value(d.body) +} + +// decodeObject decodes member i into metadata[name] and returns it when it is +// an object. The filters edit the returned map in place. +func (d *npmPackument) decodeObject(i int, metadata map[string]any, name string) map[string]any { + raw := d.value(i) + if raw == nil { + return nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil + } + metadata[name] = value + object, _ := value.(map[string]any) + return object +} + +// write assembles the document in member order, with writeValue writing each +// member's value. Earlier duplicates of the filtered keys are dropped: they +// would carry unfiltered data, and JSON.parse ignores them anyway. +func (d *npmPackument) write(writeValue func(out *bytes.Buffer, i int) error) ([]byte, error) { + var out bytes.Buffer + out.Grow(len(d.body) + len(d.body)/8) + out.WriteByte('{') + first := true + for i, m := range d.members { + if i != d.versionsAt && i != d.timeAt && i != d.tagsAt { + key := m.Key(d.body) + if jsonscan.KeyIs(key, "versions") || jsonscan.KeyIs(key, "time") || jsonscan.KeyIs(key, "dist-tags") { + continue + } + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.Key(d.body)) + out.WriteByte(':') + if err := writeValue(&out, i); err != nil { + return nil, err + } + } + out.WriteByte('}') + return out.Bytes(), nil +} + +// writeNPMVersions writes the versions object, keeping the versions still in +// keep and pointing each kept version's tarball at this proxy, followed by the +// pushed versions in local. +func (h *transformer) writeNPMVersions(out *bytes.Buffer, packageName string, versionsRaw []byte, keep map[string]any, local []LocalVersion) error { + out.WriteByte('{') + first := true + rewritten := 0 + err := jsonscan.EachMember(versionsRaw, func(m jsonscan.Member) error { + version, err := jsonscan.Key(m.Key(versionsRaw)) + if err != nil { + return err + } + if _, ok := keep[version]; !ok { + return nil + } + if !first { + out.WriteByte(',') + } + first = false + out.Write(m.Key(versionsRaw)) + out.WriteByte(':') + if h.writeNPMVersion(out, packageName, version, m.Value(versionsRaw)) { + rewritten++ + } + return nil + }) + h.tarballsRewritten(rewritten) + for _, e := range local { + if !first { + out.WriteByte(',') + } + first = false + key, _ := json.Marshal(e.Version) + out.Write(key) + out.WriteByte(':') + out.Write(e.Entry) + } + out.WriteByte('}') + return err +} + +// writeNPMVersion writes one version entry with its dist.tarball pointing at +// this proxy, and reports whether it rewrote the tarball. An entry without a +// string tarball is written unchanged. +func (h *transformer) writeNPMVersion(out *bytes.Buffer, packageName, version string, raw []byte) bool { + dist, ok, err := jsonscan.FindMember(raw, "dist") + if err != nil || !ok { + out.Write(raw) + return false + } + distRaw := dist.Value(raw) + tarball, ok, err := jsonscan.FindMember(distRaw, "tarball") + if err != nil || !ok || distRaw[tarball.ValStart] != '"' { + out.Write(raw) + return false + } + var oldTarball string + if err := json.Unmarshal(tarball.Value(distRaw), &oldTarball); err != nil { + out.Write(raw) + return false + } + + newTarball := TarballURL(h.ProxyURL, packageName, version, oldTarball) + encoded, err := json.Marshal(newTarball) + if err != nil { + out.Write(raw) + return false + } + out.Write(raw[:dist.ValStart+tarball.ValStart]) + out.Write(encoded) + out.Write(raw[dist.ValStart+tarball.ValEnd:]) + return true +} + +// tarballsRewritten reports once per document how many tarball URLs it +// rewrote. A report per version flooded debug logs: packages such as eslint +// list hundreds of versions, and with cooldown on every request rewrites the +// document again. +func (h *transformer) tarballsRewritten(count int) { + if count > 0 && h.OnTarballsRewritten != nil { + h.OnTarballsRewritten(count) + } +} + +// applyCooldownFiltering removes versions that are too recently published, +// and updates dist-tags.latest if the current latest was filtered out. +func (h *transformer) applyCooldownFiltering(metadata map[string]any, versions map[string]any, packageName string) { + if h.Cooldown == nil || !h.Cooldown.Enabled() { + return + } + + timeMap, _ := metadata["time"].(map[string]any) + if timeMap == nil { + return + } + + packagePURL := packageurl.MakeString("npm", packageName, "") + + for version := range versions { + publishedStr, ok := timeMap[version].(string) + if !ok { + continue + } + publishedAt, err := time.Parse(time.RFC3339, publishedStr) + if err != nil { + continue + } + if !h.Cooldown.Evaluate("npm", packagePURL, publishedAt, h.EvaluatedAt).Allowed { + if h.OnCooldownFilter != nil { + h.OnCooldownFilter(version, publishedStr) + } + delete(versions, version) + delete(timeMap, version) + } + } + + h.updateDistTagsLatest(metadata, versions, timeMap) +} + +// updateDistTagsLatest updates the dist-tags.latest field if the current latest +// version was removed by cooldown filtering. +func (h *transformer) updateDistTagsLatest(metadata, versions, timeMap map[string]any) { + distTags, ok := metadata["dist-tags"].(map[string]any) + if !ok { + return + } + + latest, ok := distTags["latest"].(string) + if !ok { + return + } + + if _, exists := versions[latest]; exists { + return + } + + if newLatest := h.findNewestVersion(versions, timeMap); newLatest != "" { + distTags["latest"] = newLatest + } +} + +// TarballURL returns the proxy URL that serves the tarball upstream +// lists at tarball for this version. +func TarballURL(proxyURL, packageName, version, tarball string) string { + filename := tarball + if idx := strings.LastIndex(tarball, "/"); idx >= 0 { + filename = tarball[idx+1:] + } + if VersionFromFilename(packageName, filename) != version { + _, shortName, scoped := strings.Cut(packageName, "/") + if !scoped { + shortName = packageName + } + filename = shortName + "-" + version + ".tgz" + } + + return fmt.Sprintf("%s/npm/%s/-/%s", proxyURL, url.PathEscape(packageName), filename) +} + +// findNewestVersion returns the version string with the most recent timestamp +// from the remaining versions, using the time map. +func (h *transformer) findNewestVersion(versions map[string]any, timeMap map[string]any) string { + if timeMap == nil { + return "" + } + + type versionTime struct { + version string + t time.Time + } + + var vts []versionTime + for v := range versions { + if ts, ok := timeMap[v].(string); ok { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + vts = append(vts, versionTime{v, t}) + } + } + } + + if len(vts) == 0 { + return "" + } + + sort.Slice(vts, func(i, j int) bool { + return vts[i].t.After(vts[j].t) + }) + + return vts[0].version +} + +// VersionFromFilename extracts a version from an npm tarball filename. +func VersionFromFilename(packageName, filename string) string { + // Remove .tgz extension + if !strings.HasSuffix(filename, ".tgz") { + return "" + } + base := strings.TrimSuffix(filename, ".tgz") + + // For scoped packages, the filename uses the short name + shortName := packageName + if _, name, scoped := strings.Cut(packageName, "/"); scoped { + shortName = name + } + + // Expected format: {shortName}-{version} + prefix := shortName + "-" + if !strings.HasPrefix(base, prefix) { + return "" + } + + return strings.TrimPrefix(base, prefix) +} diff --git a/internal/metadata/npm/metadata_test.go b/internal/metadata/npm/metadata_test.go new file mode 100644 index 00000000..5d68dd64 --- /dev/null +++ b/internal/metadata/npm/metadata_test.go @@ -0,0 +1,167 @@ +package npm + +import ( + "encoding/json" + "fmt" + "github.com/git-pkgs/cooldown" + "strings" + "testing" + "time" +) + +// TestNPMRewriteMetadataKeepsUpstreamBytes checks that only tarball URLs +// change: everything else, including numbers encoding/json would round +// through float64 and characters it would HTML-escape, is copied verbatim. +func TestNPMRewriteMetadataKeepsUpstreamBytes(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{"name":"demo","_big":12345678901234567890,"readme":"&",` + + `"versions":{"1.0.0":{"version":"1.0.0","dist":{"shasum":"s","tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz","integrity":"i"}}}}` + + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + want := strings.Replace(input, "https://registry.npmjs.org/demo/-/demo-1.0.0.tgz", "http://proxy/npm/demo/-/demo-1.0.0.tgz", 1) + if string(out) != want { + t.Errorf("rewrite =\n%s\nwant\n%s", out, want) + } +} + +func TestNPMRewriteMetadataUnusualEntries(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{ + "versions": { + "1.0.0": "not an object", + "2.0.0": {"dist": "no object either"}, + "3.0.0": {"dist": {"tarball": 7}}, + "4.0.0": {"dist": {"tarball": "https://registry.npmjs.org/demo/-/renamed.tgz"}}, + "5.0.0": {"dist": {}} + } + }` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + var doc struct { + Versions map[string]json.RawMessage `json:"versions"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + for v, want := range map[string]string{ + "1.0.0": `"not an object"`, + "2.0.0": `{"dist": "no object either"}`, + "3.0.0": `{"dist": {"tarball": 7}}`, + "4.0.0": `{"dist": {"tarball": "http://proxy/npm/demo/-/demo-4.0.0.tgz"}}`, + "5.0.0": `{"dist": {}}`, + } { + if got := string(doc.Versions[v]); got != want { + t.Errorf("%s = %s, want %s", v, got, want) + } + } +} + +func TestNPMRewriteMetadataInvalidJSON(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + for _, input := range []string{`{"versions": {}`, `[1]`, `{"versions": {}} trailing`} { + if _, _, err := Rewrite("demo", []byte(input), options); err == nil { + t.Errorf("%s: expected an error", input) + } + } +} + +func TestNPMRewriteMetadataNoVersions(t *testing.T) { + options := Options{ProxyURL: "http://proxy"} + input := `{"name":"demo","versions":[]}` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil || string(out) != input { + t.Errorf("rewrite = %s, %v; want input unchanged", out, err) + } + + options.DeniedVersions = map[string]bool{"1.0.0": true} + if _, _, err := Rewrite("demo", []byte(input), options); err == nil { + t.Error("expected an error when a denylisted package has no versions object") + } +} + +// TestNPMRewriteMetadataCooldownAndDenylist runs both filters together and +// checks that versions, their time entries and the tags pointing at them are +// all removed, with latest moved to the newest version left. +func TestNPMRewriteMetadataCooldownAndDenylist(t *testing.T) { + now := time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC) + ts := func(age time.Duration) string { return now.Add(-age).UTC().Format(time.RFC3339) } + + options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} + options.Cooldown = &cooldown.Config{Default: "3d"} + options.DeniedVersions = map[string]bool{"2.0.0": true} + + version := func(v string) string { + return fmt.Sprintf(`%q:{"version":%q,"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-%s.tgz"}}`, v, v, v) + } + input := `{"name":"demo",` + + `"dist-tags":{"latest":"3.0.0","beta":"2.0.0","next":"1.0.0"},` + + `"versions":{` + version("1.0.0") + `,` + version("2.0.0") + `,` + version("3.0.0") + `},` + + `"time":{"created":"` + ts(1000*time.Hour) + `","1.0.0":"` + ts(900*time.Hour) + `","2.0.0":"` + ts(800*time.Hour) + `","3.0.0":"` + ts(time.Hour) + `"}}` + + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + var doc struct { + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("output is not JSON: %v\n%s", err, out) + } + if len(doc.Versions) != 1 || doc.Versions["1.0.0"] == nil { + t.Errorf("versions = %v, want only 1.0.0", doc.Versions) + } + if _, ok := doc.Time["2.0.0"]; ok { + t.Error("time still lists denylisted 2.0.0") + } + if _, ok := doc.Time["3.0.0"]; ok { + t.Error("time still lists 3.0.0, which is inside cooldown") + } + if _, ok := doc.Time["created"]; !ok { + t.Error("time lost its created entry") + } + want := map[string]string{"latest": "1.0.0", "next": "1.0.0"} + if fmt.Sprint(doc.DistTags) != fmt.Sprint(want) { + t.Errorf("dist-tags = %v, want %v", doc.DistTags, want) + } + if !strings.Contains(string(doc.Versions["1.0.0"]), "http://proxy/npm/demo/-/demo-1.0.0.tgz") { + t.Errorf("1.0.0 tarball not rewritten: %s", doc.Versions["1.0.0"]) + } +} + +// TestNPMRewriteMetadataDuplicateKeys checks that a repeated top-level key +// cannot smuggle an unfiltered copy of the versions past the filters. +func TestNPMRewriteMetadataDuplicateKeys(t *testing.T) { + options := Options{ProxyURL: "http://proxy", EvaluatedAt: time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)} + options.DeniedVersions = map[string]bool{"2.0.0": true} + input := `{"versions":{"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}},` + + `"versions":{"1.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-1.0.0.tgz"}},"2.0.0":{"dist":{"tarball":"https://registry.npmjs.org/demo/-/demo-2.0.0.tgz"}}}}` + out, _, err := Rewrite("demo", []byte(input), options) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "2.0.0") || strings.Count(string(out), `"versions"`) != 1 { + t.Errorf("rewrite = %s", out) + } +} + +func TestAbbreviated(t *testing.T) { + for body, want := range map[string]bool{ + `{"name":"demo","modified":"2024-01-01T00:00:00Z","versions":{}}`: true, + `{"name":"demo","modified":"2024-01-01T00:00:00Z","time":{},"versions":{}}`: false, + `{"name":"demo","versions":{}}`: false, + `["modified"]`: false, + `{"modified":`: false, + } { + if got := Abbreviated([]byte(body)); got != want { + t.Errorf("Abbreviated(%s) = %v, want %v", body, got, want) + } + } +} diff --git a/internal/metadata/npm/policy_test.go b/internal/metadata/npm/policy_test.go new file mode 100644 index 00000000..7726d294 --- /dev/null +++ b/internal/metadata/npm/policy_test.go @@ -0,0 +1,161 @@ +package npm + +import ( + "bytes" + _ "embed" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +//go:embed testdata/packument.json +var packument []byte + +func TestRewriteCallbacks(t *testing.T) { + options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} + want, kept, err := Rewrite("@example/widgets", packument, options) + if err != nil { + t.Fatal(err) + } + var rewrites, filtered int + options.OnTarballsRewritten = func(count int) { + rewrites++ + if count != 1 { + t.Errorf("rewrite count = %d, want 1", count) + } + } + options.OnCooldownFilter = func(version, publishedAt string) { + filtered++ + if version != "3.0.0" || publishedAt != "2099-01-01T00:00:00Z" { + t.Errorf("filter event: %s, %s", version, publishedAt) + } + } + out, gotKept, err := Rewrite("@example/widgets", packument, options) + if err != nil || !bytes.Equal(out, want) || !slices.Equal(gotKept, kept) || rewrites != 1 || filtered != 1 { + t.Fatalf("callbacks: rewrites=%d, filtered=%d, kept=%v, error=%v", rewrites, filtered, gotKept, err) + } +} + +func TestRewriteHostedOverride(t *testing.T) { + options := Options{ + ProxyURL: "https://proxy.example", + Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, + EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC), + LocalVersions: []LocalVersion{{Version: "3.0.0", PublishedAt: "2030-01-01T00:00:00Z", + Entry: json.RawMessage(`{"version":"3.0.0","dist":{"tarball":"https://proxy.example/npm/@example%2Fwidgets/-/widgets-3.0.0.tgz","integrity":"sha512-local"}}`)}}, + } + before := bytes.Clone(packument) + localBefore := bytes.Clone(options.LocalVersions[0].Entry) + out, kept, err := Rewrite("@example/widgets", packument, options) + if err != nil { + t.Fatal(err) + } + slices.Sort(kept) + if !slices.Equal(kept, []string{"1.0.0", "3.0.0"}) { + t.Fatalf("kept = %v", kept) + } + var doc struct { + Versions map[string]json.RawMessage `json:"versions"` + Tags map[string]string `json:"dist-tags"` + Time map[string]string `json:"time"` + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Versions) != 2 || !bytes.Equal(doc.Versions["3.0.0"], localBefore) || doc.Time["3.0.0"] != options.LocalVersions[0].PublishedAt { + t.Fatalf("hosted override lost: %s", out) + } + if len(doc.Tags) != 1 || doc.Tags["latest"] != "1.0.0" || doc.Time["2.0.0"] != "" || doc.Time["created"] == "" { + t.Fatalf("policy output: %s", out) + } + for _, want := range []string{"12345678901234567890", "Widgets & tools", "https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"} { + if !strings.Contains(string(out), want) { + t.Errorf("output lost %q", want) + } + } + if !bytes.Equal(packument, before) || !bytes.Equal(options.LocalVersions[0].Entry, localBefore) || len(options.DeniedVersions) != 1 || !options.DeniedVersions["2.0.0"] { + t.Fatal("rewrite mutated its input") + } + again, _, err := Rewrite("@example/widgets", packument, options) + if err != nil || !bytes.Equal(out, again) { + t.Fatalf("repeat changed output: %v", err) + } +} + +func TestVersionPolicyErrors(t *testing.T) { + options := Options{ProxyURL: "https://proxy.example", Cooldown: &cooldown.Config{Default: "7d"}, + DeniedVersions: map[string]bool{"2.0.0": true}, EvaluatedAt: time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC)} + for _, tc := range []struct { + specifier string + want error + }{ + {"1.0.0", nil}, {"latest", nil}, {"2.0.0", ErrVersionDenied}, {"3.0.0", ErrVersionNotFound}, {"beta", ErrVersionNotFound}, {"missing", ErrVersionNotFound}, + } { + t.Run(tc.specifier, func(t *testing.T) { + var counts []int + options := options + options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } + out, err := Version("@example/widgets", tc.specifier, packument, options) + if !errors.Is(err, tc.want) { + t.Fatalf("error = %v, want %v", err, tc.want) + } + // The packument is reported once; picking one version out of it + // is not a second report. + if !slices.Equal(counts, []int{1}) { + t.Errorf("rewrite reports = %v, want [1]", counts) + } + if tc.want == nil && (!bytes.Contains(out, []byte(`"version": "1.0.0"`)) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz"))) { + t.Fatalf("resolved version: %s", out) + } + }) + } +} + +func TestRewriteCooldownEvaluationTime(t *testing.T) { + body := []byte(`{"versions":{"1.0.0":{}},"time":{"1.0.0":"2030-01-01T00:00:00Z"}}`) + boundary := time.Date(2030, 1, 8, 0, 0, 0, 0, time.UTC) + options := Options{Cooldown: &cooldown.Config{Default: "7d"}} + for _, tc := range []struct { + at time.Time + count int + }{ + {boundary.Add(-time.Nanosecond), 0}, {boundary, 1}, {boundary.Add(time.Second), 1}, + } { + options.EvaluatedAt = tc.at + _, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || len(kept) != tc.count { + t.Errorf("at %s: kept %v, error %v", tc.at, kept, err) + } + } + options.Cooldown = &cooldown.Config{Default: "7d", Packages: map[string]string{"pkg:npm/%40example/widgets": "0"}} + options.EvaluatedAt = boundary.Add(-time.Hour) + _, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || len(kept) != 1 { + t.Fatalf("scoped exemption: kept %v, error %v", kept, err) + } +} + +func TestRewriteVersionDocument(t *testing.T) { + body := []byte(`{"name":"@example/widgets","version":"1.0.0","dist":{"tarball":"https://cdn.example/abcdef.tgz"}}`) + options := Options{ProxyURL: "https://proxy.example"} + out, kept, err := Rewrite("@example/widgets", body, options) + if err != nil || !slices.Equal(kept, []string{"1.0.0"}) || !bytes.Contains(out, []byte("https://proxy.example/npm/@example%2Fwidgets/-/widgets-1.0.0.tgz")) { + t.Fatalf("rewrite: %s, %v, %v", out, kept, err) + } + var counts []int + options.OnTarballsRewritten = func(count int) { counts = append(counts, count) } + if _, _, err := Rewrite("@example/widgets", body, options); err != nil || !slices.Equal(counts, []int{1}) { + t.Fatalf("version document reported %v, %v", counts, err) + } + options.DeniedVersions = map[string]bool{"1.0.0": true} + if _, _, err := Rewrite("@example/widgets", body, options); !errors.Is(err, ErrVersionDenied) { + t.Fatalf("denied version: %v", err) + } +} diff --git a/internal/metadata/npm/testdata/packument.json b/internal/metadata/npm/testdata/packument.json new file mode 100644 index 00000000..cdb36b55 --- /dev/null +++ b/internal/metadata/npm/testdata/packument.json @@ -0,0 +1,12 @@ +{ + "name": "@example/widgets", + "dist-tags": {"latest": "3.0.0", "beta": "2.0.0"}, + "versions": { + "1.0.0": {"name": "@example/widgets", "version": "1.0.0", "dependencies": {"left-pad": "^1.3.0"}, "dist": {"integrity": "sha512-example", "tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-1.0.0.tgz"}}, + "2.0.0": {"name": "@example/widgets", "version": "2.0.0", "dist": {"tarball": "https://registry.npmjs.org/@example/widgets/-/widgets-2.0.0.tgz"}}, + "3.0.0": {"name": "@example/widgets", "version": "3.0.0", "dist": {"tarball": "https://cdn.example/abcdef.tgz"}} + }, + "time": {"created": "2020-01-01T00:00:00Z", "1.0.0": "2020-01-01T00:00:00Z", "2.0.0": "2021-01-01T00:00:00Z", "3.0.0": "2099-01-01T00:00:00Z"}, + "readme": "Widgets & tools", + "downloads": 12345678901234567890 +}