List mirror versions from the configured upstream

cmd/proxy/main.go

@@ -558,6 +558,13 @@ func runMirror() {
 
 	m := mirror.New(proxy, db, store, logger, *concurrency)
 	m.Images = images
+	m.Versions = server.NewMirrorVersions(cfg.Upstream, proxy.HTTPClient)
+	switch s := source.(type) {
+	case *mirror.PURLSource:
+		s.Versions = m.Versions
+	case *mirror.SBOMSource:
+		s.Versions = m.Versions
+	}
 	if cfg.Scanning.Enabled {
 		refuseUnscannedMirroring(m)
 	}

docs/configuration.md

@@ -1522,6 +1522,10 @@ Positional arguments are treated as PURLs:
 proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
 ```
 
+### Unversioned PURLs
+
+A PURL without a version, here or in a mirror API job, mirrors every version the configured upstream lists, through the same client, so `upstream.auth` and `upstream.allow_private_hosts` apply. npm packages in `upstream.npm_routes` are listed from their route. Listing works for npm, gem, golang, hex (from `upstream.hex_api`), pub, pypi, maven, nuget, composer, cran and brew. For cargo, conda and julia the upstream doesn't serve the API used for listing, so versions come from the public registry while the upstream is the default one; with another upstream, and for Composer packages in `upstream.composer_routes` or any other ecosystem, give a version.
+
 ### Container images
 
 `pkg:docker` and `pkg:oci` PURLs mirror a container image the way a pull does: the manifest or index, the platform manifests, then config and layer blobs. Everything is stored under the keys a pull through `/v2` reads, so a later `docker pull` or containerd pull is served from cache, also when the upstream is unreachable.

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/git-pkgs/integrity v0.2.0
 	github.com/git-pkgs/magic v0.5.0
 	github.com/git-pkgs/purl v0.1.24
-	github.com/git-pkgs/registries v0.11.0
+	github.com/git-pkgs/registries v0.11.1
 	github.com/git-pkgs/sbom v0.1.7
 	github.com/git-pkgs/spdx v0.3.3
 	github.com/go-chi/chi/v5 v5.5.0

go.sum

@@ -135,8 +135,8 @@ github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8=
 github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw=
 github.com/git-pkgs/purl v0.1.24 h1:JMEB1uNWI+baEpl6AeVx+Kfq1c+xkou+THxa5KJ3MrU=
 github.com/git-pkgs/purl v0.1.24/go.mod h1:l8rCfvt5iXVrFIcoQ8+vCLOaHhrtEW2id0LENP+IzMw=
-github.com/git-pkgs/registries v0.11.0 h1:kkB80NjO9UncyyxGfKmdPXWqUMNE/CkNw5Z/6gqW7Rs=
-github.com/git-pkgs/registries v0.11.0/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc=
+github.com/git-pkgs/registries v0.11.1 h1:AkVQ15Rnrho4tbTnocYuHQuzNQ1m9jEjWKTA8P3xnPI=
+github.com/git-pkgs/registries v0.11.1/go.mod h1:bKxgdy/PwnzQXL2FnOktmSQHTKk6B85MA4VtvvYItPc=
 github.com/git-pkgs/sbom v0.1.7 h1:4eboy/Yed/SC9ZcC/xT50iywHPzWbmCU7azDb5T1RbA=
 github.com/git-pkgs/sbom v0.1.7/go.mod h1:KIkZX0v31l62SeFsi2qsQXcZRVrAXvIdPQ4x4en+G2M=
 github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI=

internal/mirror/job.go

@@ -209,9 +209,9 @@ func (js *JobStore) sourceFromRequest(req JobRequest) (Source, error) { //nolint
 	case len(req.PURLs) > 0 && len(req.SBOM) > 0:
 		return nil, fmt.Errorf("request must include only one of purls or sbom")
 	case len(req.PURLs) > 0:
-		return &PURLSource{PURLs: req.PURLs, Platforms: req.Platforms}, nil
+		return &PURLSource{PURLs: req.PURLs, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
 	case len(req.SBOM) > 0:
-		return &SBOMSource{Data: req.SBOM, Platforms: req.Platforms}, nil
+		return &SBOMSource{Data: req.SBOM, Versions: js.mirror.Versions, Platforms: req.Platforms}, nil
 	case req.Registry != "":
 		return nil, fmt.Errorf("registry mirroring is not yet implemented; use purls instead")
 	default:

internal/mirror/mirror.go

@@ -27,6 +27,10 @@ type Mirror struct {
 	// other unsupported ecosystem.
 	Images ImageMirrorer
 
+	// Versions lists the versions of unversioned PURLs in mirror jobs and
+	// schedules. Nil lists them from the public registries.
+	Versions VersionLister
+
 	// RefusePackages, when set, fails every item Images does not take with
 	// this reason instead of fetching it, so nothing is cached.
 	RefusePackages string

internal/mirror/sbom_version_test.go

@@ -40,7 +40,7 @@ func TestSBOMSourceVersionPrecedence(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			data := marshalJSON(t, map[string]any{"components": []map[string]any{{"name": "app", "version": test.componentVersion, "purl": test.imagePURL}}})
 			var got []PackageVersion
-			source := &SBOMSource{Data: data, RegClient: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}
+			source := &SBOMSource{Data: data, Versions: RegistryVersions{Client: registries.NewClient(registries.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}))}}
 			if err := source.Enumerate(context.Background(), func(pv PackageVersion) error { got = append(got, pv); return nil }); err != nil {
 				t.Fatal(err)
 			}

internal/mirror/schedule.go

@@ -102,7 +102,7 @@ func (s *Scheduler) run(ctx context.Context, index int, schedule Schedule) {
 		}
 	}
 	started := time.Now()
-	progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Platforms: schedule.Platforms})
+	progress, err := s.mirror.Run(ctx, &PURLSource{PURLs: schedule.PURLs, Versions: s.mirror.Versions, Platforms: schedule.Platforms})
 	if ctx.Err() != nil {
 		if s.Claim != nil && s.Release != nil {
 			if err := s.Release(key); err != nil {

internal/mirror/source.go

@@ -5,8 +5,6 @@ import (
 	"fmt"
 
 	"github.com/git-pkgs/purl"
-	"github.com/git-pkgs/registries"
-	_ "github.com/git-pkgs/registries/all"
 	"github.com/git-pkgs/sbom"
 )
 
@@ -44,18 +42,21 @@ type Source interface {
 }
 
 // PURLSource yields packages from PURL strings.
-// Versioned PURLs produce a single item. Unversioned PURLs look up all versions from the registry.
+// Versioned PURLs produce a single item. Unversioned PURLs produce one item
+// per version Versions lists.
 type PURLSource struct {
-	PURLs     []string
-	RegClient *registries.Client
+	PURLs []string
+	// Versions lists the versions of unversioned PURLs. Nil lists them from
+	// the public registries.
+	Versions VersionLister
 	// Platforms limit which platforms of container images are mirrored.
 	Platforms []string
 }
 
 func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) error) error {
-	client := s.RegClient
-	if client == nil {
-		client = registries.DefaultClient()
+	versions := s.Versions
+	if versions == nil {
+		versions = RegistryVersions{}
 	}
 
 	for _, purlStr := range s.PURLs {
@@ -85,11 +86,11 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 		}
 
 		// Unversioned: enumerate all versions
-		versions, err := s.fetchVersions(ctx, client, ecosystem, name)
+		listed, err := versions.ListVersions(ctx, ecosystem, name)
 		if err != nil {
 			return fmt.Errorf("fetching versions for %s/%s: %w", ecosystem, name, err)
 		}
-		for _, v := range versions {
+		for _, v := range listed {
 			if err := fn(PackageVersion{Ecosystem: ecosystem, Name: name, Version: v}); err != nil {
 				return err
 			}
@@ -98,27 +99,13 @@ func (s *PURLSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 	return nil
 }
 
-func (s *PURLSource) fetchVersions(ctx context.Context, client *registries.Client, ecosystem, name string) ([]string, error) {
-	reg, err := registries.New(purl.EcosystemToPURLType(ecosystem), "", client)
-	if err != nil {
-		return nil, err
-	}
-	versions, err := reg.FetchVersions(ctx, name)
-	if err != nil {
-		return nil, err
-	}
-	result := make([]string, len(versions))
-	for i, v := range versions {
-		result[i] = v.Number
-	}
-	return result, nil
-}
-
 // SBOMSource extracts package versions from CycloneDX or SPDX SBOM data.
 type SBOMSource struct {
-	Data      []byte
-	Name      string
-	RegClient *registries.Client
+	Data []byte
+	Name string
+	// Versions lists the versions of unversioned PURLs. Nil lists them from
+	// the public registries.
+	Versions VersionLister
 	// Platforms limit which platforms of container images are mirrored.
 	Platforms []string
 }
@@ -132,7 +119,7 @@ func (s *SBOMSource) Enumerate(ctx context.Context, fn func(PackageVersion) erro
 		return fmt.Errorf("parsing SBOM: %w", err)
 	}
 
-	inner := &PURLSource{PURLs: purls, RegClient: s.RegClient, Platforms: s.Platforms}
+	inner := &PURLSource{PURLs: purls, Versions: s.Versions, Platforms: s.Platforms}
 	return inner.Enumerate(ctx, fn)
 }
 

internal/mirror/versions.go

@@ -0,0 +1,52 @@
+package mirror
+
+import (
+	"context"
+
+	"github.com/git-pkgs/purl"
+	"github.com/git-pkgs/registries"
+	_ "github.com/git-pkgs/registries/all"
+)
+
+// VersionLister lists the versions of a package, for PURLs without one.
+type VersionLister interface {
+	ListVersions(ctx context.Context, ecosystem, name string) ([]string, error)
+}
+
+// RegistryVersions lists versions through the registries clients.
+type RegistryVersions struct {
+	// Client makes the requests. Nil uses registries.DefaultClient.
+	Client *registries.Client
+	// URL returns the registry base URL to list a package from, by PURL
+	// type. Nil, or an empty URL, uses the public registry.
+	URL func(purlType, name string) (string, error)
+}
+
+// ListVersions implements VersionLister.
+func (r RegistryVersions) ListVersions(ctx context.Context, ecosystem, name string) ([]string, error) {
+	purlType := purl.EcosystemToPURLType(ecosystem)
+	var baseURL string
+	if r.URL != nil {
+		var err error
+		if baseURL, err = r.URL(purlType, name); err != nil {
+			return nil, err
+		}
+	}
+	client := r.Client
+	if client == nil {
+		client = registries.DefaultClient()
+	}
+	reg, err := registries.New(purlType, baseURL, client)
+	if err != nil {
+		return nil, err
+	}
+	versions, err := reg.FetchVersions(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]string, len(versions))
+	for i, v := range versions {
+		result[i] = v.Number
+	}
+	return result, nil
+}

internal/mirror/versions_test.go

@@ -0,0 +1,98 @@
+package mirror
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"reflect"
+	"testing"
+)
+
+// fakeVersions lists fixed versions and records what it was asked for.
+type fakeVersions struct {
+	versions []string
+	err      error
+	asked    []string
+}
+
+func (f *fakeVersions) ListVersions(_ context.Context, ecosystem, name string) ([]string, error) {
+	f.asked = append(f.asked, ecosystem+"/"+name)
+	return f.versions, f.err
+}
+
+func enumerate(t *testing.T, source Source) []PackageVersion {
+	t.Helper()
+	var items []PackageVersion
+	if err := source.Enumerate(context.Background(), func(pv PackageVersion) error {
+		items = append(items, pv)
+		return nil
+	}); err != nil {
+		t.Fatalf("Enumerate() error = %v", err)
+	}
+	return items
+}
+
+func TestPURLSourceListsVersionsThroughLister(t *testing.T) {
+	versions := &fakeVersions{versions: []string{"1.0.0", "1.1.0"}}
+	source := &PURLSource{PURLs: []string{"pkg:npm/%40acme/widget"}, Versions: versions}
+
+	items := enumerate(t, source)
+
+	want := []PackageVersion{
+		{Ecosystem: "npm", Name: "@acme/widget", Version: "1.0.0"},
+		{Ecosystem: "npm", Name: "@acme/widget", Version: "1.1.0"},
+	}
+	if !reflect.DeepEqual(items, want) {
+		t.Errorf("items = %v, want %v", items, want)
+	}
+	if !reflect.DeepEqual(versions.asked, []string{"npm/@acme/widget"}) {
+		t.Errorf("lister asked for %v, want npm/@acme/widget", versions.asked)
+	}
+}
+
+func TestPURLSourceReportsListerError(t *testing.T) {
+	source := &PURLSource{PURLs: []string{"pkg:npm/left-pad"}, Versions: &fakeVersions{err: errors.New("upstream down")}}
+	err := source.Enumerate(context.Background(), func(PackageVersion) error { return nil })
+	if err == nil {
+		t.Fatal("Enumerate() succeeded although listing versions failed")
+	}
+}
+
+func TestSBOMSourceListsVersionsThroughLister(t *testing.T) {
+	bom := map[string]any{
+		"bomFormat":   "CycloneDX",
+		"specVersion": "1.4",
+		"components":  []map[string]any{{"type": "library", "name": "left-pad", "purl": "pkg:npm/left-pad"}},
+	}
+	versions := &fakeVersions{versions: []string{"1.3.0"}}
+	source := &SBOMSource{Data: marshalJSON(t, bom), Versions: versions}
+
+	items := enumerate(t, source)
+
+	if len(items) != 1 || items[0].Version != "1.3.0" {
+		t.Errorf("items = %v, want left-pad@1.3.0 from the lister", items)
+	}
+}
+
+func TestSourceFromRequestUsesMirrorVersions(t *testing.T) {
+	m := setupTestMirror(t, 1)
+	versions := &fakeVersions{}
+	m.Versions = versions
+	js := NewJobStore(context.Background(), m)
+
+	source, err := js.sourceFromRequest(JobRequest{PURLs: []string{"pkg:npm/left-pad"}})
+	if err != nil {
+		t.Fatalf("sourceFromRequest() error = %v", err)
+	}
+	if got := source.(*PURLSource).Versions; got != versions {
+		t.Errorf("PURL source lists versions through %v, want the mirror's lister", got)
+	}
+
+	source, err = js.sourceFromRequest(JobRequest{SBOM: json.RawMessage(`{"bomFormat":"CycloneDX","components":[]}`)})
+	if err != nil {
+		t.Fatalf("sourceFromRequest() error = %v", err)
+	}
+	if got := source.(*SBOMSource).Versions; got != versions {
+		t.Errorf("SBOM source lists versions through %v, want the mirror's lister", got)
+	}
+}

internal/server/mirror_schedule.go

@@ -17,8 +17,7 @@ func (s *Server) startMirrorSchedules(ctx context.Context, proxy *handler.Proxy)
 		return
 	}
 	// One worker, so a schedule never fans out against the registry.
-	m := mirror.New(proxy, s.db, s.storage, s.logger, 1)
-	m.Images = s.newContainerHandler(proxy)
+	m := s.newMirror(proxy, 1)
 	schedules := make([]mirror.Schedule, len(s.cfg.Mirror.Schedules))
 	for i, schedule := range s.cfg.Mirror.Schedules {
 		schedules[i] = mirror.Schedule{PURLs: schedule.Images, Platforms: schedule.Platforms, Interval: schedule.IntervalDuration()}