Add an Alpine APK repository client for v2 indexes

README.md

@@ -1,6 +1,6 @@
 # registries
 
-Go library for fetching package metadata from registry APIs. Supports 27 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`).
+Go library for fetching package metadata from registry APIs. Supports 28 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`).
 
 ## Installation
 
@@ -134,6 +134,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5)
 | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` |
 | RubyGems | `pkg:gem/rails@7.1.0` |
 | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` |
+| Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` |
 
 ## Direct Registry Usage
 
@@ -193,6 +194,9 @@ import _ "github.com/git-pkgs/registries/all"
 | Deno | `deno` | https://apiland.deno.dev |
 | Terraform | `terraform` | https://registry.terraform.io |
 | Swift | `swift` | URL required |
+| Alpine | `apk` | URL required |
+
+The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index.
 
 ### Swift registries
 

all/all.go

@@ -9,10 +9,11 @@
 //
 //	// Now all ecosystems are available
 //	ecosystems := registries.SupportedEcosystems()
-//	// ["brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"]
+//	// ["apk", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"]
 package all
 
 import (
+	_ "github.com/git-pkgs/registries/internal/apk"
 	_ "github.com/git-pkgs/registries/internal/cargo"
 	_ "github.com/git-pkgs/registries/internal/clojars"
 	_ "github.com/git-pkgs/registries/internal/cocoapods"

client/client.go

@@ -84,7 +84,7 @@ func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) {
 
 func (c *Client) getBody(ctx context.Context, url string, headers http.Header) ([]byte, http.Header, error) {
 	var body []byte
-	resp, responseHeaders, err := c.open(ctx, url, headers, func(resp *http.Response) error {
+	resp, responseHeaders, err := c.open(ctx, http.MethodGet, url, headers, func(resp *http.Response) error {
 		defer func() { _ = resp.Body.Close() }()
 		var err error
 		body, err = readLimited(resp.Body, url, c.maxResponseSize())
@@ -102,7 +102,7 @@ func (c *Client) getBody(ctx context.Context, url string, headers http.Header) (
 // while fn reads the body is returned as is. fn must consume the complete
 // JSON document; trailing data and body read errors are rejected.
 func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decoder) error) error {
-	resp, _, err := c.open(ctx, url, nil, nil)
+	resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil)
 	if err != nil {
 		return err
 	}
@@ -179,7 +179,7 @@ func (l *limitedReader) Read(p []byte) (int, error) {
 // open retries requests and optional body reads within one retry budget.
 // read must close the body; a nil read leaves it open for the caller. On
 // failure it returns the headers of the last response received, if any.
-func (c *Client) open(ctx context.Context, url string, headers http.Header, read func(*http.Response) error) (*http.Response, http.Header, error) {
+func (c *Client) open(ctx context.Context, method, url string, headers http.Header, read func(*http.Response) error) (*http.Response, http.Header, error) {
 	var lastErr error
 	var lastHeaders http.Header
 
@@ -202,7 +202,7 @@ func (c *Client) open(ctx context.Context, url string, headers http.Header, read
 			}
 		}
 
-		resp, responseHeaders, err := c.doRequest(ctx, url, headers)
+		resp, responseHeaders, err := c.doRequest(ctx, method, url, headers)
 		if err == nil && read != nil {
 			err = read(resp)
 		}
@@ -235,8 +235,8 @@ func (c *Client) open(ctx context.Context, url string, headers http.Header, read
 	return nil, lastHeaders, lastErr
 }
 
-func (c *Client) doRequest(ctx context.Context, url string, headers http.Header) (*http.Response, http.Header, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+func (c *Client) doRequest(ctx context.Context, method, url string, headers http.Header) (*http.Response, http.Header, error) {
+	req, err := http.NewRequestWithContext(ctx, method, url, nil)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -297,22 +297,21 @@ func (c *Client) GetText(ctx context.Context, url string) (string, error) {
 	return string(body), nil
 }
 
-// Head sends a HEAD request and returns the status code.
+// Head sends a HEAD request and returns the status code, with the same retry
+// and rate-limit policy as GetBody. An HTTP error status is returned as the
+// status code, not an error.
 func (c *Client) Head(ctx context.Context, url string) (int, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
-	if err != nil {
-		return 0, err
+	resp, _, err := c.open(ctx, http.MethodHead, url, nil, func(resp *http.Response) error {
+		return resp.Body.Close()
+	})
+	if err == nil {
+		return resp.StatusCode, nil
 	}
-
-	req.Header.Set("User-Agent", c.UserAgent)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return 0, err
+	var httpErr *HTTPError
+	if isHTTPError(err, &httpErr) {
+		return httpErr.StatusCode, nil
 	}
-	_ = resp.Body.Close()
-
-	return resp.StatusCode, nil
+	return 0, err
 }
 
 // WithRateLimiter returns a copy of the client with the given rate limiter.

client/client_test.go

@@ -253,3 +253,43 @@ func TestResponseWithinLimit(t *testing.T) {
 		t.Fatalf("got %q, want %q", string(body), payload)
 	}
 }
+
+type countingLimiter struct{ waits int }
+
+func (l *countingLimiter) Wait(context.Context) error {
+	l.waits++
+	return nil
+}
+
+func TestHeadRetriesAndRateLimits(t *testing.T) {
+	var methods []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		methods = append(methods, r.Method)
+		switch {
+		case r.URL.Path == "/missing":
+			w.WriteHeader(http.StatusNotFound)
+		case len(methods) == 1:
+			w.WriteHeader(http.StatusBadGateway)
+		}
+	}))
+	defer srv.Close()
+
+	limiter := &countingLimiter{}
+	c := NewClient(WithMaxRetries(1)).WithRateLimiter(limiter)
+	c.BaseDelay = 0
+
+	status, err := c.Head(context.Background(), srv.URL)
+	if err != nil || status != http.StatusOK {
+		t.Errorf("Head = %d, %v; want 200 after a retry", status, err)
+	}
+	status, err = c.Head(context.Background(), srv.URL+"/missing")
+	if err != nil || status != http.StatusNotFound {
+		t.Errorf("Head missing = %d, %v; want 404 and no error", status, err)
+	}
+	if strings.Join(methods, " ") != "HEAD HEAD HEAD" {
+		t.Errorf("requests = %v", methods)
+	}
+	if limiter.waits != 3 {
+		t.Errorf("rate limiter waits = %d, want 3", limiter.waits)
+	}
+}

fetch/apk_resolver_test.go

@@ -0,0 +1,94 @@
+package fetch
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/git-pkgs/registries"
+	_ "github.com/git-pkgs/registries/internal/apk"
+)
+
+func TestResolveAPK(t *testing.T) {
+	reg, err := registries.New("apk", "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver()
+	r.RegisterRegistry(reg)
+
+	info, err := r.Resolve(context.Background(), "apk", "alpine/curl", "8.22.0-r0")
+	if err != nil {
+		t.Fatalf("Resolve failed: %v", err)
+	}
+	if want := "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk"; info.URL != want {
+		t.Errorf("URL = %q, want %q", info.URL, want)
+	}
+	if info.Filename != "curl-8.22.0-r0.apk" {
+		t.Errorf("Filename = %q", info.Filename)
+	}
+}
+
+func TestResolveAPKArchQualifierMismatch(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasSuffix(r.URL.Path, "/APKINDEX.tar.gz") {
+			http.NotFound(w, r)
+			return
+		}
+		var buf bytes.Buffer
+		gz := gzip.NewWriter(&buf)
+		tw := tar.NewWriter(gz)
+		body := "P:curl\nV:8.22.0-r0\nA:x86_64\n"
+		_ = tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body))})
+		_, _ = tw.Write([]byte(body))
+		_ = tw.Close()
+		_ = gz.Close()
+		_, _ = w.Write(buf.Bytes())
+	}))
+	defer server.Close()
+
+	reg, err := registries.New("apk", server.URL+"/x86_64", registries.DefaultClient())
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver()
+	r.RegisterRegistry(reg)
+
+	info, err := r.ResolveWithOptions(context.Background(), "apk", "alpine/curl", "8.22.0-r0",
+		ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64"}})
+	if !errors.Is(err, ErrNoMatchingArtifact) {
+		t.Fatalf("aarch64 against x86_64 = %+v, %v; want ErrNoMatchingArtifact", info, err)
+	}
+}
+
+func TestResolveAPKRejectsPathInVersion(t *testing.T) {
+	var paths []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		paths = append(paths, r.URL.Path)
+		http.NotFound(w, r)
+	}))
+	defer server.Close()
+
+	reg, err := registries.New("apk", server.URL+"/v3.23/main/x86_64", registries.DefaultClient())
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := NewResolver()
+	r.RegisterRegistry(reg)
+
+	info, err := r.Resolve(context.Background(), "apk", "alpine/curl", "../../../edge/main/x86_64/other-1.0-r0")
+	if err == nil {
+		t.Fatalf("Resolve = %+v, want an error", info)
+	}
+	for _, p := range paths {
+		if strings.Contains(p, "edge") || strings.HasSuffix(p, ".apk") {
+			t.Errorf("requested %s", p)
+		}
+	}
+}

fetch/resolver.go

@@ -122,6 +122,15 @@ type ResolveOptions struct {
 // metadata of the same name.
 var selectionQualifiers = []string{"arch", "distro", "subdir", "build"}
 
+func (o ResolveOptions) hasSelectionQualifier() bool {
+	for _, key := range selectionQualifiers {
+		if o.Qualifiers[key] != "" {
+			return true
+		}
+	}
+	return false
+}
+
 // Resolve returns the download URL and filename for a package artifact.
 func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) {
 	return r.ResolveWithOptions(ctx, ecosystem, name, version, ResolveOptions{})
@@ -143,13 +152,17 @@ func (r *Resolver) ResolveWithOptions(
 		return matchSingleArtifact(info, options, err)
 	}
 
-	// Try the simple URL builder first
-	if url := reg.URLs().Download(name, version); url != "" {
-		info := &ArtifactInfo{
-			URL:      url,
-			Filename: filenameFromURL(url),
+	// Try the simple URL builder first. It can't honour selection
+	// qualifiers such as arch, so a request carrying one goes through
+	// the metadata path where qualifiersMatch is applied.
+	if !options.hasSelectionQualifier() {
+		if url := reg.URLs().Download(name, version); url != "" {
+			info := &ArtifactInfo{
+				URL:      url,
+				Filename: filenameFromURL(url),
+			}
+			return matchSingleArtifact(info, options, nil)
 		}
-		return matchSingleArtifact(info, options, nil)
 	}
 
 	// For ecosystems like PyPI, we need to fetch metadata to get the URL