diff --git a/README.md b/README.md index 3dcaaa5..96b429a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # registries -Go library for fetching package metadata from registry APIs. Supports 27 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`). +Go library for fetching package metadata from registry APIs. Supports 28 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`). ## Installation @@ -134,6 +134,7 @@ packages = registries.BulkFetchPackagesWithConcurrency(ctx, purls, nil, 5) | Maven | `pkg:maven/org.apache.commons/commons-lang3@3.12.0` | | RubyGems | `pkg:gem/rails@7.1.0` | | Terraform | `pkg:terraform/hashicorp/consul/aws@0.11.0` | +| Alpine | `pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64` | ## Direct Registry Usage @@ -193,6 +194,9 @@ import _ "github.com/git-pkgs/registries/all" | Deno | `deno` | https://apiland.deno.dev | | Terraform | `terraform` | https://registry.terraform.io | | Swift | `swift` | URL required | +| Alpine | `apk` | URL required | + +The Alpine URL is one repository architecture directory, such as `https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64`, or the `APKINDEX.tar.gz` inside it. The client doesn't pick a release, component or architecture from a mirror root. It reads v2 `APKINDEX.tar.gz` indexes. When a directory serves a v3 `Packages.adb`, the client returns an error rather than reading the v2 index. ### Swift registries diff --git a/all/all.go b/all/all.go index 01b9126..a45789a 100644 --- a/all/all.go +++ b/all/all.go @@ -9,10 +9,11 @@ // // // Now all ecosystems are available // ecosystems := registries.SupportedEcosystems() -// // ["brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] +// // ["apk", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"] package all import ( + _ "github.com/git-pkgs/registries/internal/apk" _ "github.com/git-pkgs/registries/internal/cargo" _ "github.com/git-pkgs/registries/internal/clojars" _ "github.com/git-pkgs/registries/internal/cocoapods" diff --git a/client/client.go b/client/client.go index 0621232..aedf379 100644 --- a/client/client.go +++ b/client/client.go @@ -84,7 +84,7 @@ func (c *Client) GetBody(ctx context.Context, url string) ([]byte, error) { func (c *Client) getBody(ctx context.Context, url string, headers http.Header) ([]byte, http.Header, error) { var body []byte - resp, responseHeaders, err := c.open(ctx, url, headers, func(resp *http.Response) error { + resp, responseHeaders, err := c.open(ctx, http.MethodGet, url, headers, func(resp *http.Response) error { defer func() { _ = resp.Body.Close() }() var err error body, err = readLimited(resp.Body, url, c.maxResponseSize()) @@ -102,7 +102,7 @@ func (c *Client) getBody(ctx context.Context, url string, headers http.Header) ( // while fn reads the body is returned as is. fn must consume the complete // JSON document; trailing data and body read errors are rejected. func (c *Client) StreamJSON(ctx context.Context, url string, fn func(*json.Decoder) error) error { - resp, _, err := c.open(ctx, url, nil, nil) + resp, _, err := c.open(ctx, http.MethodGet, url, nil, nil) if err != nil { return err } @@ -179,7 +179,7 @@ func (l *limitedReader) Read(p []byte) (int, error) { // open retries requests and optional body reads within one retry budget. // read must close the body; a nil read leaves it open for the caller. On // failure it returns the headers of the last response received, if any. -func (c *Client) open(ctx context.Context, url string, headers http.Header, read func(*http.Response) error) (*http.Response, http.Header, error) { +func (c *Client) open(ctx context.Context, method, url string, headers http.Header, read func(*http.Response) error) (*http.Response, http.Header, error) { var lastErr error var lastHeaders http.Header @@ -202,7 +202,7 @@ func (c *Client) open(ctx context.Context, url string, headers http.Header, read } } - resp, responseHeaders, err := c.doRequest(ctx, url, headers) + resp, responseHeaders, err := c.doRequest(ctx, method, url, headers) if err == nil && read != nil { err = read(resp) } @@ -235,8 +235,8 @@ func (c *Client) open(ctx context.Context, url string, headers http.Header, read return nil, lastHeaders, lastErr } -func (c *Client) doRequest(ctx context.Context, url string, headers http.Header) (*http.Response, http.Header, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) +func (c *Client) doRequest(ctx context.Context, method, url string, headers http.Header) (*http.Response, http.Header, error) { + req, err := http.NewRequestWithContext(ctx, method, url, nil) if err != nil { return nil, nil, err } @@ -297,22 +297,21 @@ func (c *Client) GetText(ctx context.Context, url string) (string, error) { return string(body), nil } -// Head sends a HEAD request and returns the status code. +// Head sends a HEAD request and returns the status code, with the same retry +// and rate-limit policy as GetBody. An HTTP error status is returned as the +// status code, not an error. func (c *Client) Head(ctx context.Context, url string) (int, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) - if err != nil { - return 0, err + resp, _, err := c.open(ctx, http.MethodHead, url, nil, func(resp *http.Response) error { + return resp.Body.Close() + }) + if err == nil { + return resp.StatusCode, nil } - - req.Header.Set("User-Agent", c.UserAgent) - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return 0, err + var httpErr *HTTPError + if isHTTPError(err, &httpErr) { + return httpErr.StatusCode, nil } - _ = resp.Body.Close() - - return resp.StatusCode, nil + return 0, err } // WithRateLimiter returns a copy of the client with the given rate limiter. diff --git a/client/client_test.go b/client/client_test.go index c28cd7c..0c377ea 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -253,3 +253,43 @@ func TestResponseWithinLimit(t *testing.T) { t.Fatalf("got %q, want %q", string(body), payload) } } + +type countingLimiter struct{ waits int } + +func (l *countingLimiter) Wait(context.Context) error { + l.waits++ + return nil +} + +func TestHeadRetriesAndRateLimits(t *testing.T) { + var methods []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method) + switch { + case r.URL.Path == "/missing": + w.WriteHeader(http.StatusNotFound) + case len(methods) == 1: + w.WriteHeader(http.StatusBadGateway) + } + })) + defer srv.Close() + + limiter := &countingLimiter{} + c := NewClient(WithMaxRetries(1)).WithRateLimiter(limiter) + c.BaseDelay = 0 + + status, err := c.Head(context.Background(), srv.URL) + if err != nil || status != http.StatusOK { + t.Errorf("Head = %d, %v; want 200 after a retry", status, err) + } + status, err = c.Head(context.Background(), srv.URL+"/missing") + if err != nil || status != http.StatusNotFound { + t.Errorf("Head missing = %d, %v; want 404 and no error", status, err) + } + if strings.Join(methods, " ") != "HEAD HEAD HEAD" { + t.Errorf("requests = %v", methods) + } + if limiter.waits != 3 { + t.Errorf("rate limiter waits = %d, want 3", limiter.waits) + } +} diff --git a/fetch/apk_resolver_test.go b/fetch/apk_resolver_test.go new file mode 100644 index 0000000..6dd6b71 --- /dev/null +++ b/fetch/apk_resolver_test.go @@ -0,0 +1,94 @@ +package fetch + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/registries" + _ "github.com/git-pkgs/registries/internal/apk" +) + +func TestResolveAPK(t *testing.T) { + reg, err := registries.New("apk", "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64", nil) + if err != nil { + t.Fatal(err) + } + r := NewResolver() + r.RegisterRegistry(reg) + + info, err := r.Resolve(context.Background(), "apk", "alpine/curl", "8.22.0-r0") + if err != nil { + t.Fatalf("Resolve failed: %v", err) + } + if want := "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk"; info.URL != want { + t.Errorf("URL = %q, want %q", info.URL, want) + } + if info.Filename != "curl-8.22.0-r0.apk" { + t.Errorf("Filename = %q", info.Filename) + } +} + +func TestResolveAPKArchQualifierMismatch(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/APKINDEX.tar.gz") { + http.NotFound(w, r) + return + } + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + body := "P:curl\nV:8.22.0-r0\nA:x86_64\n" + _ = tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body))}) + _, _ = tw.Write([]byte(body)) + _ = tw.Close() + _ = gz.Close() + _, _ = w.Write(buf.Bytes()) + })) + defer server.Close() + + reg, err := registries.New("apk", server.URL+"/x86_64", registries.DefaultClient()) + if err != nil { + t.Fatal(err) + } + r := NewResolver() + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "apk", "alpine/curl", "8.22.0-r0", + ResolveOptions{Qualifiers: map[string]string{"arch": "aarch64"}}) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Fatalf("aarch64 against x86_64 = %+v, %v; want ErrNoMatchingArtifact", info, err) + } +} + +func TestResolveAPKRejectsPathInVersion(t *testing.T) { + var paths []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + http.NotFound(w, r) + })) + defer server.Close() + + reg, err := registries.New("apk", server.URL+"/v3.23/main/x86_64", registries.DefaultClient()) + if err != nil { + t.Fatal(err) + } + r := NewResolver() + r.RegisterRegistry(reg) + + info, err := r.Resolve(context.Background(), "apk", "alpine/curl", "../../../edge/main/x86_64/other-1.0-r0") + if err == nil { + t.Fatalf("Resolve = %+v, want an error", info) + } + for _, p := range paths { + if strings.Contains(p, "edge") || strings.HasSuffix(p, ".apk") { + t.Errorf("requested %s", p) + } + } +} diff --git a/fetch/resolver.go b/fetch/resolver.go index 263b579..dbe4469 100644 --- a/fetch/resolver.go +++ b/fetch/resolver.go @@ -122,6 +122,15 @@ type ResolveOptions struct { // metadata of the same name. var selectionQualifiers = []string{"arch", "distro", "subdir", "build"} +func (o ResolveOptions) hasSelectionQualifier() bool { + for _, key := range selectionQualifiers { + if o.Qualifiers[key] != "" { + return true + } + } + return false +} + // Resolve returns the download URL and filename for a package artifact. func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) { return r.ResolveWithOptions(ctx, ecosystem, name, version, ResolveOptions{}) @@ -143,13 +152,17 @@ func (r *Resolver) ResolveWithOptions( return matchSingleArtifact(info, options, err) } - // Try the simple URL builder first - if url := reg.URLs().Download(name, version); url != "" { - info := &ArtifactInfo{ - URL: url, - Filename: filenameFromURL(url), + // Try the simple URL builder first. It can't honour selection + // qualifiers such as arch, so a request carrying one goes through + // the metadata path where qualifiersMatch is applied. + if !options.hasSelectionQualifier() { + if url := reg.URLs().Download(name, version); url != "" { + info := &ArtifactInfo{ + URL: url, + Filename: filenameFromURL(url), + } + return matchSingleArtifact(info, options, nil) } - return matchSingleArtifact(info, options, nil) } // For ecosystems like PyPI, we need to fetch metadata to get the URL diff --git a/internal/apk/apk.go b/internal/apk/apk.go new file mode 100644 index 0000000..bd7225d --- /dev/null +++ b/internal/apk/apk.go @@ -0,0 +1,586 @@ +// Package apk provides a registry client for Alpine APK repositories. +package apk + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/git-pkgs/registries/internal/core" + "github.com/git-pkgs/vers" +) + +const ( + DefaultURL = "" + ecosystem = "apk" + + indexV2 = "APKINDEX.tar.gz" + indexV3 = "Packages.adb" + + // maxIndexSize bounds the decompressed index archive. Alpine's + // community index is about 10 MB uncompressed. + maxIndexSize = 64 << 20 + maxLineSize = 1 << 20 + + archKey = "arch" +) + +var ( + // ErrV3IndexUnsupported is returned when a repository publishes a v3 + // Packages.adb index, which this client does not decode yet. + ErrV3IndexUnsupported = errors.New("apk v3 Packages.adb indexes are not supported") + // ErrMalformedIndex is returned for indexes that cannot be parsed. + ErrMalformedIndex = errors.New("malformed APKINDEX") +) + +func init() { + core.Register(ecosystem, DefaultURL, func(baseURL string, client *core.Client) core.Registry { + return New(baseURL, client) + }) +} + +// Registry reads one repository architecture directory, such as +// https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64, or an exact index +// URL inside one. The release, component and architecture are never inferred +// from a higher-level mirror URL. +// +// The index is downloaded once per Registry and kept for its lifetime, so +// callers looking up many packages should reuse one Registry per repository. +type Registry struct { + dirURL string + indexURL string // empty for a directory, which is probed + client *core.Client + urls *URLs + + mu sync.Mutex + idx *repoIndex +} + +type repoIndex struct { + packages map[string][]record + // providers maps a virtual name from p:, such as so:libz.so.1, to the + // package chosen to satisfy it. + providers map[string]string +} + +func New(baseURL string, client *core.Client) *Registry { + if client == nil { + client = core.DefaultClient() + } + baseURL = strings.TrimSuffix(baseURL, "/") + r := &Registry{dirURL: baseURL, client: client} + if strings.HasSuffix(baseURL, "/"+indexV2) || strings.HasSuffix(baseURL, "/"+indexV3) { + r.indexURL = baseURL + r.dirURL = baseURL[:strings.LastIndex(baseURL, "/")] + } + r.urls = &URLs{dirURL: r.dirURL} + return r +} + +func (r *Registry) Ecosystem() string { + return ecosystem +} + +func (r *Registry) URLs() core.URLBuilder { //nolint:ireturn + return r.urls +} + +// record is one package entry from APKINDEX. +type record struct { + name string + version string + arch string + description string + url string + license string + maintainer string + origin string + commit string + checksum string + buildTime time.Time + size int64 + installedSize int64 + providerPriority int64 + depends []string + provides []string + replaces []string + installIf []string +} + +// splitName strips a PURL vendor namespace: "alpine/curl" looks up "curl". +func splitName(name string) (namespace, pkg string) { + if i := strings.LastIndex(name, "/"); i >= 0 { + return name[:i], name[i+1:] + } + return "", name +} + +func (r *Registry) index(ctx context.Context) (*repoIndex, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.idx != nil { + return r.idx, nil + } + + body, err := r.fetchIndex(ctx) + if err != nil { + return nil, err + } + records, err := parseIndexArchive(body) + if err != nil { + return nil, err + } + + packages := make(map[string][]record) + for _, rec := range records { + packages[rec.name] = append(packages[rec.name], rec) + } + for _, recs := range packages { + sort.SliceStable(recs, func(i, j int) bool { + return vers.CompareWithScheme(recs[i].version, recs[j].version, ecosystem) > 0 + }) + } + r.idx = &repoIndex{packages: packages, providers: buildProviders(records)} + return r.idx, nil +} + +// buildProviders picks one package per virtual name: the highest k: provider +// priority, then the lowest name so the choice is stable. +func buildProviders(records []record) map[string]string { + type choice struct { + name string + priority int64 + } + best := make(map[string]choice) + for _, rec := range records { + for _, p := range rec.provides { + virtual, _ := splitConstraint(p) + cur, ok := best[virtual] + if !ok || rec.providerPriority > cur.priority || + (rec.providerPriority == cur.priority && rec.name < cur.name) { + best[virtual] = choice{rec.name, rec.providerPriority} + } + } + } + providers := make(map[string]string, len(best)) + for virtual, c := range best { + providers[virtual] = c.name + } + return providers +} + +// fetchIndex downloads APKINDEX.tar.gz. For a directory it asks for +// Packages.adb first, as apk-tools does, and falls back only when that is +// not found. +func (r *Registry) fetchIndex(ctx context.Context) ([]byte, error) { + if strings.HasSuffix(r.indexURL, "/"+indexV3) { + return nil, fmt.Errorf("%w: %s", ErrV3IndexUnsupported, r.indexURL) + } + if r.indexURL != "" { + return r.client.GetBody(ctx, r.indexURL) + } + + v3URL := r.dirURL + "/" + indexV3 + status, err := r.client.Head(ctx, v3URL) + switch { + case err != nil: + return nil, err + case status >= 200 && status < 300: + return nil, fmt.Errorf("%w: %s", ErrV3IndexUnsupported, v3URL) + case status >= http.StatusInternalServerError: + return nil, &core.HTTPError{StatusCode: status, URL: v3URL} + } + // Any 4xx falls back: 403 is how S3-backed repositories answer for a + // missing key, and the probe only asks whether a v3 index is served. + return r.client.GetBody(ctx, r.dirURL+"/"+indexV2) +} + +// parseIndexArchive reads the APKINDEX member of an APKINDEX.tar.gz. The file +// is a signature gzip stream followed by the index stream, read as one tar. +func parseIndexArchive(data []byte) ([]record, error) { + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + defer func() { _ = gz.Close() }() + + limited := &io.LimitedReader{R: gz, N: maxIndexSize + 1} + tr := tar.NewReader(limited) + var index []byte + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + if hdr.Name != "APKINDEX" || index != nil { + continue + } + index, err = io.ReadAll(tr) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + } + // tar EOF can precede gzip's checksum and size trailer. + if _, err := io.Copy(io.Discard, limited); err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + if limited.N == 0 { + return nil, fmt.Errorf("%w: index archive larger than %d bytes", ErrMalformedIndex, maxIndexSize) + } + if index == nil { + return nil, fmt.Errorf("%w: no APKINDEX member", ErrMalformedIndex) + } + return parseIndex(index) +} + +// parseIndex parses blank-line-separated records of single-letter fields, +// mapped as in apk-tools' apk_adb.c. +func parseIndex(data []byte) ([]record, error) { + var records []record + var rec record + inRecord := false + lineNo := 0 + + finish := func() error { + if !inRecord { + return nil + } + if rec.name == "" || rec.version == "" { + return fmt.Errorf("%w: record ending at line %d has no P or V field", ErrMalformedIndex, lineNo) + } + records = append(records, rec) + rec = record{} + inRecord = false + return nil + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(nil, maxLineSize) + for scanner.Scan() { + lineNo++ + line := scanner.Text() + if line == "" { + if err := finish(); err != nil { + return nil, err + } + continue + } + if len(line) < 2 || line[1] != ':' { + return nil, fmt.Errorf("%w: line %d is not a field", ErrMalformedIndex, lineNo) + } + inRecord = true + if err := rec.set(line[0], line[2:]); err != nil { + return nil, fmt.Errorf("%w: line %d: %w", ErrMalformedIndex, lineNo, err) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("%w: %w", ErrMalformedIndex, err) + } + if err := finish(); err != nil { + return nil, err + } + return records, nil +} + +func (rec *record) set(key byte, value string) error { + var err error + switch key { + case 'P': + rec.name = value + case 'V': + rec.version = value + case 'A': + rec.arch = value + case 'T': + rec.description = value + case 'U': + rec.url = value + case 'L': + rec.license = value + case 'm': + rec.maintainer = value + case 'o': + rec.origin = value + case 'c': + rec.commit = value + case 'C': + rec.checksum = value + case 'S': + rec.size, err = strconv.ParseInt(value, 10, 64) + case 'I': + rec.installedSize, err = strconv.ParseInt(value, 10, 64) + case 'k': + rec.providerPriority, err = strconv.ParseInt(value, 10, 64) + case 't': + var secs int64 + if secs, err = strconv.ParseInt(value, 10, 64); err == nil { + rec.buildTime = time.Unix(secs, 0).UTC() + } + case 'D': + rec.depends = strings.Fields(value) + case 'p': + rec.provides = strings.Fields(value) + case 'r': + rec.replaces = strings.Fields(value) + case 'i': + rec.installIf = strings.Fields(value) + } + return err +} + +// checksum is a decoded APKINDEX C: field. Q1 is the SHA-1 of the package's +// control section (its second gzip stream), not of the whole .apk file, so it +// is not usable as a download integrity value. +type checksum struct { + Algorithm string + Hex string +} + +// decodeChecksum decodes a C: value: "Q1" followed by base64 SHA-1, or a +// legacy hex MD5. +func decodeChecksum(value string) (checksum, bool) { + if b64, ok := strings.CutPrefix(value, "Q1"); ok { + raw, err := base64.StdEncoding.DecodeString(b64) + if err != nil || len(raw) != 20 { + return checksum{}, false + } + return checksum{Algorithm: "sha1", Hex: hex.EncodeToString(raw)}, true + } + if raw, err := hex.DecodeString(value); err == nil && len(raw) == 16 { + return checksum{Algorithm: "md5", Hex: value}, true + } + return checksum{}, false +} + +// parseMaintainer parses "Name "; any other value is kept whole as the name. +func parseMaintainer(value string) core.Maintainer { + if open := strings.LastIndex(value, "<"); open >= 0 && strings.HasSuffix(value, ">") { + name := strings.TrimSpace(value[:open]) + email := value[open+1 : len(value)-1] + if email != "" && strings.Contains(email, "@") && !strings.ContainsAny(email, "<> ") { + return core.Maintainer{Name: name, Email: email} + } + } + return core.Maintainer{Name: strings.TrimSpace(value)} +} + +func (r *Registry) lookup(ctx context.Context, name string) ([]record, error) { + idx, err := r.index(ctx) + if err != nil { + return nil, err + } + _, pkg := splitName(name) + recs := idx.packages[pkg] + if len(recs) == 0 { + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name} + } + return recs, nil +} + +func (r *Registry) FetchPackage(ctx context.Context, name string) (*core.Package, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + namespace, _ := splitName(name) + latest := recs[0] + return &core.Package{ + Name: latest.name, + Description: latest.description, + Homepage: latest.url, + Licenses: latest.license, + Namespace: namespace, + LatestVersion: latest.version, + Metadata: map[string]any{ + "origin": latest.origin, + archKey: latest.arch, + }, + }, nil +} + +func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Version, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + versions := make([]core.Version, 0, len(recs)) + for _, rec := range recs { + versions = append(versions, r.version(rec)) + } + return versions, nil +} + +func (r *Registry) version(rec record) core.Version { + metadata := map[string]any{ + archKey: rec.arch, + "description": rec.description, + "url": rec.url, + "maintainer": rec.maintainer, + "origin": rec.origin, + "commit": rec.commit, + "size": rec.size, + "installed_size": rec.installedSize, + "depends": rec.depends, + "provides": rec.provides, + "replaces": rec.replaces, + "install_if": rec.installIf, + } + if rec.providerPriority != 0 { + metadata["provider_priority"] = rec.providerPriority + } + if rec.checksum != "" { + metadata["checksum"] = rec.checksum + if sum, ok := decodeChecksum(rec.checksum); ok { + metadata["checksum_algorithm"] = sum.Algorithm + metadata["checksum_hex"] = sum.Hex + if sum.Algorithm == "sha1" { + metadata["checksum_covers"] = "control" + } + } + } + + v := core.Version{ + Number: rec.version, + PublishedAt: rec.buildTime, + Licenses: rec.license, + Metadata: metadata, + } + if filename, ok := packageFilename(rec.name, rec.version); ok { + v.Artifacts = []core.Artifact{{ + URL: r.dirURL + "/" + filename, + Filename: filename, + Size: rec.size, + Metadata: map[string]any{archKey: rec.arch}, + }} + } + return v +} + +// packageFilename refuses names and versions that would make the file URL +// point anywhere other than a file in the index's directory. +func packageFilename(name, version string) (string, bool) { + if name == "" || version == "" || strings.ContainsAny(name+version, `/\?#%`) { + return "", false + } + return name + "-" + version + ".apk", true +} + +func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + for _, rec := range recs { + if rec.version == version { + return r.idx.dependencies(rec), nil + } + } + return nil, &core.NotFoundError{Ecosystem: ecosystem, Name: name, Version: version} +} + +// dependencies resolves virtual names such as so:libz.so.1 to the package +// providing them. A constraint on a virtual name applies to the provided +// version, not the package's, so it is dropped. A virtual name with no +// provider in this index, usually one from another repository, is kept as is. +func (idx *repoIndex) dependencies(rec record) []core.Dependency { + deps := make([]core.Dependency, 0, len(rec.depends)) + seen := map[string]int{rec.name: -1} + for _, d := range rec.depends { + if strings.HasPrefix(d, "!") { + continue // conflict, kept in version metadata + } + depName, requirement := splitConstraint(d) + if _, ok := idx.packages[depName]; !ok { + if provider, ok := idx.providers[depName]; ok { + depName, requirement = provider, "" + } + } + // "python3 python3~3.12" is one dependency; ">=1 <2" stays as two. + if i, ok := seen[depName]; ok { + if i >= 0 && deps[i].Requirements == "" { + deps[i].Requirements = requirement + } + if i < 0 || requirement == "" || deps[i].Requirements == requirement { + continue + } + } + seen[depName] = len(deps) + deps = append(deps, core.Dependency{Name: depName, Requirements: requirement, Scope: core.Runtime}) + } + return deps +} + +// splitConstraint splits "libcurl=8.22.0-r0" into "libcurl" and "=8.22.0-r0". +func splitConstraint(dep string) (name, requirement string) { + if i := strings.IndexAny(dep, "<>=~"); i > 0 { + return dep[:i], dep[i:] + } + return dep, "" +} + +func (r *Registry) FetchMaintainers(ctx context.Context, name string) ([]core.Maintainer, error) { + recs, err := r.lookup(ctx, name) + if err != nil { + return nil, err + } + if recs[0].maintainer == "" { + return nil, nil + } + return []core.Maintainer{parseMaintainer(recs[0].maintainer)}, nil +} + +type URLs struct { + dirURL string +} + +func (u *URLs) Registry(name, version string) string { + return "" +} + +// Download returns the package file beside the index. The name may carry a +// PURL vendor namespace. +func (u *URLs) Download(name, version string) string { + _, pkg := splitName(name) + filename, ok := packageFilename(pkg, version) + if !ok { + return "" + } + return u.dirURL + "/" + filename +} + +func (u *URLs) Documentation(name, version string) string { + return "" +} + +func (u *URLs) PURL(name, version string) string { + namespace, pkg := splitName(name) + p := "pkg:apk/" + if namespace != "" { + p += namespace + "/" + } + p += pkg + if version != "" { + p += "@" + version + } + if u.dirURL != "" { + p += "?repository_url=" + url.QueryEscape(u.dirURL) + } + return p +} diff --git a/internal/apk/apk_test.go b/internal/apk/apk_test.go new file mode 100644 index 0000000..6e35d44 --- /dev/null +++ b/internal/apk/apk_test.go @@ -0,0 +1,578 @@ +package apk + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/git-pkgs/registries/internal/core" +) + +// buildIndex packs APKINDEX text the way Alpine does: a signature gzip +// stream holding an unterminated tar, then a stream with DESCRIPTION and +// APKINDEX. +func buildIndex(t *testing.T, index string) []byte { + t.Helper() + var out bytes.Buffer + + sig := gzip.NewWriter(&out) + sigTar := tar.NewWriter(sig) + writeMember(t, sigTar, ".SIGN.RSA.alpine-devel@lists.alpinelinux.org-6165ee59.rsa.pub", "signature") + if err := sigTar.Flush(); err != nil { + t.Fatal(err) + } + if err := sig.Close(); err != nil { + t.Fatal(err) + } + + body := gzip.NewWriter(&out) + bodyTar := tar.NewWriter(body) + writeMember(t, bodyTar, "DESCRIPTION", "v3.23.6-70-g8c6e6d800ba") + writeMember(t, bodyTar, "APKINDEX", index) + if err := bodyTar.Close(); err != nil { + t.Fatal(err) + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func writeMember(t *testing.T, tw *tar.Writer, name, content string) { + t.Helper() + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(content))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatal(err) + } +} + +func realIndex(t *testing.T) string { + t.Helper() + data, err := os.ReadFile("testdata/APKINDEX") + if err != nil { + t.Fatal(err) + } + return string(data) +} + +// repoServer serves a v2 repository per architecture directory, with no +// Packages.adb. +func repoServer(t *testing.T, indexes map[string]string) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + dir, file, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + index, ok := indexes[dir] + if !ok || file != indexV2 { + http.NotFound(w, r) + return + } + _, _ = w.Write(buildIndex(t, index)) + })) + t.Cleanup(server.Close) + return server +} + +func TestFetchVersionsRealIndex(t *testing.T) { + server := repoServer(t, map[string]string{"x86_64": realIndex(t)}) + reg := New(server.URL+"/x86_64", core.DefaultClient()) + + versions, err := reg.FetchVersions(context.Background(), "alpine/curl") + if err != nil { + t.Fatalf("FetchVersions failed: %v", err) + } + if len(versions) != 1 { + t.Fatalf("got %d versions, want 1", len(versions)) + } + v := versions[0] + if v.Number != "8.22.0-r0" || v.Licenses != "curl" { + t.Errorf("version = %+v", v) + } + if !v.PublishedAt.Equal(time.Unix(1788539493, 0)) { + t.Errorf("PublishedAt = %v", v.PublishedAt) + } + if v.Integrity != "" { + t.Errorf("Integrity = %q, want empty: Q1 covers only the control section", v.Integrity) + } + wantMeta := map[string]any{ + "checksum": "Q1Je9Cwt4OmEyiSGWGtYfOVCrvgWE=", + "checksum_algorithm": "sha1", + "checksum_hex": "25ef42c2de0e984ca2486586b587ce542aef8161", + "checksum_covers": "control", + "origin": "curl", + "commit": "ce63a8e353eacc32ad36fd49ae1f427f4306da87", + "arch": "x86_64", + "installed_size": int64(285023), + "size": int64(178681), + } + for k, want := range wantMeta { + if v.Metadata[k] != want { + t.Errorf("Metadata[%s] = %#v, want %#v", k, v.Metadata[k], want) + } + } + provides, _ := v.Metadata["provides"].([]string) + if strings.Join(provides, " ") != "wcurl=8.22.0-r0 cmd:curl=8.22.0-r0 cmd:wcurl=8.22.0-r0" { + t.Errorf("provides = %v", provides) + } + + a := v.Artifacts[0] + if a.URL != server.URL+"/x86_64/curl-8.22.0-r0.apk" || a.Filename != "curl-8.22.0-r0.apk" || a.Size != 178681 || a.Metadata["arch"] != "x86_64" { + t.Errorf("artifact = %+v", a) + } + + doc, err := reg.FetchVersions(context.Background(), "curl-doc") + if err != nil { + t.Fatalf("FetchVersions curl-doc failed: %v", err) + } + installIf, _ := doc[0].Metadata["install_if"].([]string) + if strings.Join(installIf, " ") != "docs curl=8.22.0-r0" { + t.Errorf("install_if = %v", installIf) + } + + sevenZip, err := reg.FetchVersions(context.Background(), "7zip") + if err != nil { + t.Fatalf("FetchVersions 7zip failed: %v", err) + } + if sevenZip[0].Metadata["provider_priority"] != int64(100) { + t.Errorf("provider_priority = %#v", sevenZip[0].Metadata["provider_priority"]) + } +} + +func TestFetchPackageKeepsNamespace(t *testing.T) { + server := repoServer(t, map[string]string{"x86_64": realIndex(t)}) + reg := New(server.URL+"/x86_64/", core.DefaultClient()) + + pkg, err := reg.FetchPackage(context.Background(), "alpine/curl") + if err != nil { + t.Fatalf("FetchPackage failed: %v", err) + } + if pkg.Name != "curl" || pkg.Namespace != "alpine" || pkg.LatestVersion != "8.22.0-r0" || + pkg.Homepage != "https://curl.se/" || pkg.Description != "URL retrieval utility and library" { + t.Errorf("package = %+v", pkg) + } + + bare, err := reg.FetchPackage(context.Background(), "curl") + if err != nil || bare.Namespace != "" { + t.Errorf("bare name package = %+v, %v", bare, err) + } +} + +func TestFetchDependencies(t *testing.T) { + index := realIndex(t) + "P:withconflict\nV:1.0-r0\nD:busybox>=1.36 !curl pc:zlib~1.3 so:libc.musl-x86_64.so.1\n" + server := repoServer(t, map[string]string{"x86_64": index}) + reg := New(server.URL+"/x86_64", core.DefaultClient()) + + deps, err := reg.FetchDependencies(context.Background(), "alpine/curl", "8.22.0-r0") + if err != nil { + t.Fatalf("FetchDependencies failed: %v", err) + } + var got []string + for _, d := range deps { + if d.Scope != core.Runtime { + t.Errorf("%s scope = %q, want runtime", d.Name, d.Scope) + } + got = append(got, d.Name+d.Requirements) + } + if strings.Join(got, " ") != "libcurl=8.22.0-r0 so:libc.musl-x86_64.so.1 so:libcurl.so.4 so:libz.so.1" { + t.Errorf("deps = %v", got) + } + + deps, err = reg.FetchDependencies(context.Background(), "withconflict", "1.0-r0") + if err != nil { + t.Fatalf("FetchDependencies failed: %v", err) + } + got = nil + for _, d := range deps { + got = append(got, d.Name+"|"+d.Requirements) + } + if strings.Join(got, " ") != "busybox|>=1.36 pc:zlib|~1.3 so:libc.musl-x86_64.so.1|" { + t.Errorf("deps = %v, want conflicts left out", got) + } + + var notFound *core.NotFoundError + if _, err := reg.FetchDependencies(context.Background(), "curl", "8.21.0-r0"); !errors.As(err, ¬Found) || notFound.Version != "8.21.0-r0" { + t.Errorf("missing version err = %v", err) + } +} + +func TestFetchDependenciesResolvesProviders(t *testing.T) { + index := realIndex(t) + + "P:libcurl\nV:8.22.0-r0\nD:so:libc.musl-x86_64.so.1 so:libz.so.1\np:so:libcurl.so.4=4.8.0\n\n" + + "P:musl\nV:1.2.5-r10\np:so:libc.musl-x86_64.so.1=1\n\n" + + "P:zlib\nV:1.3.1-r2\np:so:libz.so.1=1.3.1\n\n" + + "P:zlib-ng-compat\nV:2.2.4-r0\nk:10\np:so:libz.so.1=1.3.1\n\n" + + "P:samepriority\nV:1.0-r0\nD:cmd:tool\n\n" + + "P:tool-b\nV:1.0-r0\np:cmd:tool\n\n" + + "P:tool-a\nV:1.0-r0\np:cmd:tool\n\n" + + "P:python3\nV:3.12.12-r0\n\n" + + "P:py3-requests\nV:2.33.1-r0\nD:python3 python3~3.12 zlib>=1 zlib<2 py3-requests\n" + server := repoServer(t, map[string]string{"x86_64": index}) + reg := New(server.URL+"/x86_64", core.DefaultClient()) + + tests := map[string]string{ + "curl": "libcurl|=8.22.0-r0 musl| zlib-ng-compat|", + "libcurl": "musl| zlib-ng-compat|", + "samepriority": "tool-a|", + "busybox": "musl|", + "py3-requests": "python3|~3.12 zlib|>=1 zlib|<2", + } + for name, want := range tests { + versions, err := reg.FetchVersions(context.Background(), name) + if err != nil { + t.Fatal(err) + } + deps, err := reg.FetchDependencies(context.Background(), name, versions[0].Number) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + var got []string + for _, d := range deps { + got = append(got, d.Name+"|"+d.Requirements) + } + if strings.Join(got, " ") != want { + t.Errorf("%s deps = %v, want %s", name, got, want) + } + } +} + +func TestExactVersionsWithRevisions(t *testing.T) { + index := "P:pkg\nV:1.2.3-r9\nA:x86_64\n\nP:pkg\nV:1.2.3-r10\nA:x86_64\n\nP:pkg\nV:1.2.3_rc1-r0\nA:x86_64\n" + server := repoServer(t, map[string]string{"x86_64": index}) + reg := New(server.URL+"/x86_64", core.DefaultClient()) + + versions, err := reg.FetchVersions(context.Background(), "pkg") + if err != nil { + t.Fatalf("FetchVersions failed: %v", err) + } + var got []string + for _, v := range versions { + got = append(got, v.Number) + } + if strings.Join(got, " ") != "1.2.3-r10 1.2.3-r9 1.2.3_rc1-r0" { + t.Errorf("versions = %v, want apk ordering", got) + } + if _, err := reg.FetchDependencies(context.Background(), "pkg", "1.2.3-r9"); err != nil { + t.Errorf("exact -r9 lookup failed: %v", err) + } + if _, err := reg.FetchDependencies(context.Background(), "pkg", "1.2.3"); err == nil { + t.Error("1.2.3 matched without its revision") + } +} + +func TestTwoArchitectures(t *testing.T) { + server := repoServer(t, map[string]string{ + "x86_64": "P:curl\nV:8.22.0-r0\nA:x86_64\nS:100\n", + "aarch64": "P:curl\nV:8.22.0-r0\nA:aarch64\nS:200\n", + }) + for arch, size := range map[string]int64{"x86_64": 100, "aarch64": 200} { + versions, err := New(server.URL+"/"+arch, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatalf("%s: %v", arch, err) + } + a := versions[0].Artifacts[0] + if a.Metadata["arch"] != arch || a.Size != size || !strings.Contains(a.URL, "/"+arch+"/") { + t.Errorf("%s artifact = %+v", arch, a) + } + } +} + +func TestExactIndexURL(t *testing.T) { + var requests []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.URL.Path) + _, _ = w.Write(buildIndex(t, "P:curl\nV:8.22.0-r0\n")) + })) + defer server.Close() + + reg := New(server.URL+"/v3.23/main/x86_64/APKINDEX.tar.gz", core.DefaultClient()) + versions, err := reg.FetchVersions(context.Background(), "curl") + if err != nil { + t.Fatalf("FetchVersions failed: %v", err) + } + if want := server.URL + "/v3.23/main/x86_64/curl-8.22.0-r0.apk"; versions[0].Artifacts[0].URL != want { + t.Errorf("URL = %q, want %q", versions[0].Artifacts[0].URL, want) + } + if _, err := reg.FetchPackage(context.Background(), "curl"); err != nil { + t.Fatal(err) + } + if strings.Join(requests, " ") != "/v3.23/main/x86_64/APKINDEX.tar.gz" { + t.Errorf("requests = %v, want one index fetch with no Packages.adb probe", requests) + } +} + +func TestV3Index(t *testing.T) { + var probes []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/"+indexV3) { + probes = append(probes, r.Method) + _, _ = w.Write([]byte("ADB.indx")) + return + } + _, _ = w.Write(buildIndex(t, "P:curl\nV:8.22.0-r0\n")) + })) + defer server.Close() + + for _, base := range []string{server.URL + "/x86_64", server.URL + "/x86_64/Packages.adb"} { + _, err := New(base, core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrV3IndexUnsupported) { + t.Errorf("%s: err = %v, want ErrV3IndexUnsupported", base, err) + } + } + if strings.Join(probes, " ") != http.MethodHead { + t.Errorf("Packages.adb requests = %v, want one HEAD", probes) + } +} + +func TestProbeStatusHandling(t *testing.T) { + tests := []struct { + status int + fallback bool + }{ + {http.StatusForbidden, true}, + {http.StatusGone, true}, + {http.StatusMethodNotAllowed, true}, + {http.StatusInternalServerError, false}, + } + for _, tt := range tests { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/"+indexV3) { + w.WriteHeader(tt.status) + return + } + _, _ = w.Write(buildIndex(t, "P:curl\nV:8.22.0-r0\n")) + })) + + client := core.DefaultClient() + client.MaxRetries = 0 + _, err := New(server.URL+"/x86_64", client).FetchVersions(context.Background(), "curl") + var httpErr *core.HTTPError + if tt.fallback && err != nil { + t.Errorf("probe %d: err = %v, want fallback to APKINDEX.tar.gz", tt.status, err) + } + if !tt.fallback && (!errors.As(err, &httpErr) || httpErr.StatusCode != tt.status) { + t.Errorf("probe %d: err = %v, want the probe error surfaced", tt.status, err) + } + server.Close() + } +} + +func TestMalformedIndexes(t *testing.T) { + valid := buildIndex(t, realIndex(t)) + badChecksum := append([]byte(nil), valid...) + badChecksum[len(badChecksum)-8] ^= 1 + tests := map[string][]byte{ + "not gzip": []byte("not an index"), + "truncated": valid[:len(valid)/2], + "missing gzip footer": valid[:len(valid)-8], + "corrupt gzip checksum": badChecksum, + "no APKINDEX member": tarGz(t, "DESCRIPTION", "x"), + "line without a field": buildIndex(t, "P:curl\nV:1.0-r0\nnonsense\n"), + "record without name": buildIndex(t, "V:1.0-r0\nA:x86_64\n"), + "bad size": buildIndex(t, "P:curl\nV:1.0-r0\nS:big\n"), + "bad build time": buildIndex(t, "P:curl\nV:1.0-r0\nt:yesterday\n"), + } + for name, body := range tests { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, indexV2) { + _, _ = w.Write(body) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + _, err := New(server.URL+"/x86_64", core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedIndex) { + t.Errorf("err = %v, want ErrMalformedIndex", err) + } + }) + } +} + +func tarGz(t *testing.T, name, content string) []byte { + t.Helper() + var out bytes.Buffer + gz := gzip.NewWriter(&out) + tw := tar.NewWriter(gz) + writeMember(t, tw, name, content) + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +} + +func TestTrailingIndexDataSizeLimit(t *testing.T) { + body := bytes.NewBuffer(buildIndex(t, "P:curl\nV:1.0-r0\n")) + gz := gzip.NewWriter(body) + if _, err := io.Copy(gz, strings.NewReader(strings.Repeat(" ", maxIndexSize))); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(body.Bytes()) + })) + defer server.Close() + + _, err := New(server.URL+"/APKINDEX.tar.gz", core.DefaultClient()).FetchVersions(context.Background(), "curl") + if !errors.Is(err, ErrMalformedIndex) || !strings.Contains(err.Error(), "larger than") { + t.Errorf("err = %v, want index archive size limit error", err) + } +} + +func TestMissingOptionalFields(t *testing.T) { + server := repoServer(t, map[string]string{"x86_64": "P:minimal\nV:0.1-r0\n"}) + reg := New(server.URL+"/x86_64", core.DefaultClient()) + + versions, err := reg.FetchVersions(context.Background(), "minimal") + if err != nil { + t.Fatalf("FetchVersions failed: %v", err) + } + v := versions[0] + if !v.PublishedAt.IsZero() || v.Licenses != "" || v.Metadata["checksum"] != nil { + t.Errorf("version = %+v", v) + } + if deps, err := reg.FetchDependencies(context.Background(), "minimal", "0.1-r0"); err != nil || len(deps) != 0 { + t.Errorf("deps = %v, %v", deps, err) + } + if m, err := reg.FetchMaintainers(context.Background(), "minimal"); err != nil || m != nil { + t.Errorf("maintainers = %v, %v", m, err) + } +} + +func TestNotFoundAndCaching(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path == "/x86_64/"+indexV2 { + _, _ = w.Write(buildIndex(t, "P:curl\nV:8.22.0-r0\n")) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + reg := New(server.URL+"/x86_64", core.DefaultClient()) + var notFound *core.NotFoundError + for range 3 { + if _, err := reg.FetchVersions(context.Background(), "alpine/missing"); !errors.As(err, ¬Found) { + t.Fatalf("err = %v, want NotFoundError", err) + } + } + if requests != 2 { + t.Errorf("requests = %d, want the index probed and fetched once", requests) + } + + missingRepo := New(server.URL+"/nope/x86_64/APKINDEX.tar.gz", core.DefaultClient()) + if _, err := missingRepo.FetchVersions(context.Background(), "curl"); err == nil { + t.Error("expected an error for a missing index") + } +} + +func TestMaintainers(t *testing.T) { + tests := map[string]core.Maintainer{ + "Achill Gilgenast ": {Name: "Achill Gilgenast", Email: "achill@achill.org"}, + "Alex Xu (Hello71) ": {Name: "Alex Xu (Hello71)", Email: "alex_y_xu@yahoo.ca"}, + "Sören Tempel ": {Name: "Sören Tempel", Email: "soeren+alpine@soeren-tempel.net"}, + "Natanael Copa": {Name: "Natanael Copa"}, + "broken ": {Name: "broken "}, + "": {Email: "only@example.com"}, + } + for value, want := range tests { + if got := parseMaintainer(value); got != want { + t.Errorf("parseMaintainer(%q) = %+v, want %+v", value, got, want) + } + } + + server := repoServer(t, map[string]string{"x86_64": realIndex(t)}) + m, err := New(server.URL+"/x86_64", core.DefaultClient()).FetchMaintainers(context.Background(), "alpine/busybox") + if err != nil || len(m) != 1 || m[0].Email != "soeren+alpine@soeren-tempel.net" { + t.Errorf("FetchMaintainers = %+v, %v", m, err) + } +} + +func TestDecodeChecksum(t *testing.T) { + sum, ok := decodeChecksum("Q1Je9Cwt4OmEyiSGWGtYfOVCrvgWE=") + if !ok || sum.Algorithm != "sha1" || sum.Hex != "25ef42c2de0e984ca2486586b587ce542aef8161" { + t.Errorf("Q1 = %+v, %v", sum, ok) + } + sum, ok = decodeChecksum("d41d8cd98f00b204e9800998ecf8427e") + if !ok || sum.Algorithm != "md5" { + t.Errorf("md5 = %+v, %v", sum, ok) + } + for _, bad := range []string{"Q1notbase64!", "Q1AAAA", "abc", ""} { + if _, ok := decodeChecksum(bad); ok { + t.Errorf("decodeChecksum(%q) ok, want false", bad) + } + } +} + +type tokenTransport struct{ base http.RoundTripper } + +func (t tokenTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.Header.Set("Authorization", "Bearer secret") + return t.base.RoundTrip(req) +} + +func TestUsesSuppliedHTTPClient(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer secret" { + w.WriteHeader(http.StatusUnauthorized) + return + } + if strings.HasSuffix(r.URL.Path, indexV2) { + _, _ = w.Write(buildIndex(t, "P:curl\nV:8.22.0-r0\n")) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + client := core.DefaultClient() + client.HTTPClient = &http.Client{Transport: tokenTransport{base: http.DefaultTransport}} + if _, err := New(server.URL+"/x86_64", client).FetchVersions(context.Background(), "curl"); err != nil { + t.Errorf("FetchVersions through authenticated client: %v", err) + } +} + +func TestURLs(t *testing.T) { + u := New("https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64", nil).URLs() + tests := [][2]string{ + {u.Download("alpine/curl", "8.22.0-r0"), "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk"}, + {u.Download("curl", "8.22.0-r0"), "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk"}, + {u.Download("curl", ""), ""}, + {u.Download("curl", "../../edge/main/x86_64/other-1.0-r0"), ""}, + {u.Download("curl", "1.0-r0?x"), ""}, + {u.Download("curl", "1.0-r0%2F"), ""}, + {u.Download(`cu\rl`, "1.0-r0"), ""}, + {u.PURL("alpine/curl", "8.22.0-r0"), "pkg:apk/alpine/curl@8.22.0-r0?repository_url=https%3A%2F%2Fdl-cdn.alpinelinux.org%2Falpine%2Fv3.23%2Fmain%2Fx86_64"}, + {u.PURL("curl", ""), "pkg:apk/curl?repository_url=https%3A%2F%2Fdl-cdn.alpinelinux.org%2Falpine%2Fv3.23%2Fmain%2Fx86_64"}, + {u.Registry("curl", ""), ""}, + } + for _, tt := range tests { + if tt[0] != tt[1] { + t.Errorf("got %q, want %q", tt[0], tt[1]) + } + } + + exact := New("https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/APKINDEX.tar.gz", nil).URLs() + if got := exact.Download("curl", "8.22.0-r0"); got != "https://dl-cdn.alpinelinux.org/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk" { + t.Errorf("exact index Download = %q", got) + } +} diff --git a/internal/apk/testdata/APKINDEX b/internal/apk/testdata/APKINDEX new file mode 100644 index 0000000..f0d97be --- /dev/null +++ b/internal/apk/testdata/APKINDEX @@ -0,0 +1,64 @@ +C:Q1LSohzMjVRmn08215M0mwFPRtsg8= +P:7zip +V:25.01-r0 +A:x86_64 +S:915485 +I:1784184 +T:File archiver with a high compression ratio +U:https://7-zip.org/ +L:LGPL-2.0-only +o:7zip +m:Alex Xu (Hello71) +t:1754258820 +c:2dcce3f385a48d09a9253131950c95b1ee208189 +k:100 +D:so:libc.musl-x86_64.so.1 so:libgcc_s.so.1 so:libstdc++.so.6 +p:7zip-virtual p7zip=25.01-r0 cmd:7z=25.01-r0 cmd:7zz=25.01-r0 + +C:Q18TR4AbuWsapA0X+CI3w/T/AqRyU= +P:busybox +V:1.37.0-r30 +A:x86_64 +S:504726 +I:813480 +T:Size optimized toolbox of many common UNIX utilities +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:Sören Tempel +t:1765894768 +c:1e823a60eb85606954b3a5af5f8e5bbd1ea680cf +D:so:libc.musl-x86_64.so.1 +p:cmd:busybox=1.37.0-r30 + +C:Q1Je9Cwt4OmEyiSGWGtYfOVCrvgWE= +P:curl +V:8.22.0-r0 +A:x86_64 +S:178681 +I:285023 +T:URL retrieval utility and library +U:https://curl.se/ +L:curl +o:curl +m:Achill Gilgenast +t:1788539493 +c:ce63a8e353eacc32ad36fd49ae1f427f4306da87 +D:libcurl=8.22.0-r0 so:libc.musl-x86_64.so.1 so:libcurl.so.4 so:libz.so.1 +p:wcurl=8.22.0-r0 cmd:curl=8.22.0-r0 cmd:wcurl=8.22.0-r0 + +C:Q1PQpUpa883t9yRUqystyVqUNSz2g= +P:curl-doc +V:8.22.0-r0 +A:x86_64 +S:769930 +I:723586 +T:URL retrieval utility and library (documentation) +U:https://curl.se/ +L:curl +o:curl +m:Achill Gilgenast +t:1788539493 +c:ce63a8e353eacc32ad36fd49ae1f427f4306da87 +i:docs curl=8.22.0-r0 + diff --git a/registries_test.go b/registries_test.go index 04f0c68..6111177 100644 --- a/registries_test.go +++ b/registries_test.go @@ -1,6 +1,9 @@ package registries_test import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "encoding/json" "net/http" @@ -17,7 +20,7 @@ import ( func TestSupportedEcosystems(t *testing.T) { ecosystems := registries.SupportedEcosystems() - expected := []string{"brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} + expected := []string{"apk", "brew", "cargo", "clojars", "cocoapods", "composer", "conda", "cpan", "cran", "deno", "dub", "elm", "gem", "golang", "hackage", "haxelib", "helm", "hex", "julia", "luarocks", "maven", "nimble", "npm", "nuget", "pub", "pypi", "swift", "terraform"} sort.Strings(ecosystems) if len(ecosystems) != len(expected) { @@ -63,6 +66,7 @@ func TestNew(t *testing.T) { {"terraform", false}, {"helm", true}, {"swift", true}, + {"apk", true}, {"unknown", true}, } @@ -108,6 +112,7 @@ func TestDefaultURL(t *testing.T) { {"terraform", "https://registry.terraform.io"}, {"helm", ""}, {"swift", ""}, + {"apk", ""}, } for _, tt := range tests { @@ -280,3 +285,57 @@ func TestRootClientOptions(t *testing.T) { t.Errorf("WithSafeHTTP error %v should mention loopback", err) } } + +func TestAPKFromPURL(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/alpine/v3.23/main/x86_64/APKINDEX.tar.gz" { + http.NotFound(w, r) + return + } + _, _ = w.Write(apkIndex(t, "P:curl\nV:8.22.0-r0\nA:x86_64\nt:1788539493\nD:libcurl=8.22.0-r0\n")) + })) + defer srv.Close() + + purl := "pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64&repository_url=" + srv.URL + "/alpine/v3.23/main/x86_64" + v, err := registries.FetchVersionFromPURL(context.Background(), purl, nil) + if err != nil { + t.Fatalf("FetchVersionFromPURL failed: %v", err) + } + if v.Number != "8.22.0-r0" || v.Artifacts[0].URL != srv.URL+"/alpine/v3.23/main/x86_64/curl-8.22.0-r0.apk" { + t.Errorf("version = %+v", v) + } + + pkg, err := registries.FetchPackageFromPURL(context.Background(), purl, nil) + if err != nil || pkg.Namespace != "alpine" || pkg.Name != "curl" { + t.Errorf("package = %+v, %v", pkg, err) + } + + deps, err := registries.FetchDependenciesFromPURL(context.Background(), purl, nil) + if err != nil || len(deps) != 1 || deps[0].Name != "libcurl" { + t.Errorf("deps = %+v, %v", deps, err) + } + + if _, err := registries.FetchVersionFromPURL(context.Background(), "pkg:apk/alpine/curl@8.22.0-r0?arch=x86_64", nil); err == nil || !strings.Contains(err.Error(), "no registry URL configured") { + t.Errorf("missing URL err = %v", err) + } +} + +func apkIndex(t *testing.T, index string) []byte { + t.Helper() + var out bytes.Buffer + gz := gzip.NewWriter(&out) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(index))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(index)); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return out.Bytes() +}