Enable storage retention for conan

docs/configuration.md

@@ -91,7 +91,7 @@ A package rule wins over an ecosystem rule, which wins over `default`. Package r
 | `alpine` | yes | `pkg:apk/curl` or `pkg:apk/alpine/curl`; case doesn't matter. A rule covers the package in every configured APK repository. Alpine mirrors keep only the current build of each package per release, so an evicted older one can't be fetched again. |
 | `cargo` | yes | `pkg:cargo/serde`, the crate name as published; case, `-` and `_` have to match, so `pkg:cargo/inflector` doesn't cover `Inflector`. |
 | `composer` | yes | `pkg:composer/symfony/console`; case doesn't matter. A key without a vendor is refused. |
-| `conan` | no | - |
+| `conan` | yes | `pkg:conan/zlib`, the recipe name; a rule covers every version, revision and user/channel of it. |
 | `conda` | no | - |
 | `cran` | yes | `pkg:cran/ggplot2`, the package name as published, case included (`pkg:cran/Rcpp`). A rule covers source and binary builds alike. |
 | `deb` | no | - |

internal/handler/conan.go

@@ -3,12 +3,23 @@ package handler
 import (
 	"net/http"
 	"strings"
+
+	"github.com/git-pkgs/proxy/internal/retention"
 )
 
 const (
 	conanUpstream = "https://center.conan.io"
 )
 
+// Recipe and package archives are stored under pkg:conan/{name}, with the
+// version as the version and the revisions and any user/channel in the
+// filename, and purl keeps conan names as they are. A package rule therefore
+// covers every version, revision and user/channel of a recipe, and uses the
+// default key mapping.
+func init() {
+	retention.Default.Register(retention.Spec{Key: "conan", CanonicalPackage: retention.DefaultCanonical("conan")})
+}
+
 // ConanHandler handles Conan registry protocol requests.
 type ConanHandler struct {
 	proxy       *Proxy

internal/server/retention_conan_test.go

@@ -0,0 +1,215 @@
+package server
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/config"
+	"github.com/git-pkgs/proxy/internal/database"
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/proxy/internal/retention"
+	"github.com/git-pkgs/registries/fetch"
+)
+
+const conanRecipeRevision = "b3b71bfe8dd07abc7b82ff2bd0eac021"
+
+// conanFile is an archive as the Conan 2 client requests it and the name
+// the handler caches it under.
+type conanFile struct {
+	path     string
+	filename string
+}
+
+var (
+	conanZlibSources = conanFile{
+		"/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz",
+		conanRecipeRevision + "_conan_sources.tgz",
+	}
+	conanZlibPackage = conanFile{
+		"/v2/conans/zlib/1.3.1/_/_/revisions/" + conanRecipeRevision + "/packages/0123abc/revisions/4567def/files/conan_package.tgz",
+		conanRecipeRevision + "_0123abc_4567def_conan_package.tgz",
+	}
+	conanZlibAcme = conanFile{
+		"/v2/conans/zlib/1.2.13/acme/stable/revisions/" + conanRecipeRevision + "/files/conan_export.tgz",
+		"acme@stable_" + conanRecipeRevision + "_conan_export.tgz",
+	}
+	conanOpenSSLSources = conanFile{
+		"/v2/conans/openssl/3.3.2/_/_/revisions/" + conanRecipeRevision + "/files/conan_sources.tgz",
+		"",
+	}
+)
+
+// conanRetentionUpstream serves the archives the way a Conan 2 server
+// does, counting downloads.
+func conanRetentionUpstream(t *testing.T) (*httptest.Server, *atomic.Int64) {
+	t.Helper()
+	var downloads atomic.Int64
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for _, f := range []conanFile{conanZlibSources, conanZlibPackage, conanZlibAcme, conanOpenSSLSources} {
+			if r.URL.Path == f.path {
+				downloads.Add(1)
+				_, _ = io.WriteString(w, "conan archive bytes")
+				return
+			}
+		}
+		http.NotFound(w, r)
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &downloads
+}
+
+func newConanRetentionProxy(t *testing.T, upstream *httptest.Server) (*httptest.Server, *database.DB) {
+	t.Helper()
+	db, store := setupEvictionTest(t)
+	// The upstream is a plain-HTTP loopback server; the default fetcher
+	// refuses loopback, a stock client reaches it.
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
+	t.Cleanup(func() { _ = fetcher.Close() })
+	proxy := handler.NewProxy(db, store, fetcher, fetch.NewResolver(), discardLogger())
+	proxy.HTTPClient = upstream.Client()
+	h := handler.NewConanHandlerWithUpstream(proxy, "http://proxy.test", upstream.URL)
+	srv := httptest.NewServer(h.Routes())
+	t.Cleanup(srv.Close)
+	return srv, db
+}
+
+func conanGet(t *testing.T, srv *httptest.Server, f conanFile) {
+	t.Helper()
+	resp, err := http.Get(srv.URL + f.path)
+	if err != nil {
+		t.Fatalf("GET %s: %v", f.path, err)
+	}
+	body, _ := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s = %d %s, want 200", f.path, resp.StatusCode, body)
+	}
+}
+
+// conanArtifact returns the record of the archive of package that the
+// handler cached under filename.
+func conanArtifact(t *testing.T, db *database.DB, packagePURL, filename string) database.Artifact {
+	t.Helper()
+	var art database.Artifact
+	query := db.Rebind(`SELECT a.* FROM artifacts a JOIN versions v ON v.purl = a.version_purl
+		WHERE a.filename = ? AND v.package_purl = ?`)
+	if err := db.Get(&art, query, filename, packagePURL); err != nil {
+		t.Fatalf("reading %s of %s: %v", filename, packagePURL, err)
+	}
+	return art
+}
+
+func conanBackdateAll(t *testing.T, db *database.DB, fetched, accessed time.Time) {
+	t.Helper()
+	query := db.Rebind(`UPDATE artifacts SET fetched_at = ?, last_accessed_at = ?`)
+	if _, err := db.Exec(query, fetched, accessed); err != nil {
+		t.Fatalf("backdating: %v", err)
+	}
+}
+
+func TestConanRegistersForRetention(t *testing.T) {
+	if _, ok := retention.Default.Lookup("conan"); !ok {
+		t.Fatal("conan is not registered for retention")
+	}
+	if key, ok := retention.Default.KeyForDBEcosystem("conan"); !ok || key != "conan" {
+		t.Errorf("KeyForDBEcosystem(conan) = %q, %v, want conan, true", key, ok)
+	}
+}
+
+// A package archive served from the cache stays while it is used, is
+// evicted once its rule runs out, comes back from upstream on the next
+// download, and stays after that even with the evicted copy's old access
+// time on the record.
+func TestConanRetentionEvictsAndRefetches(t *testing.T) {
+	upstream, downloads := conanRetentionUpstream(t)
+	srv, db := newConanRetentionProxy(t, upstream)
+	cached := func() database.Artifact {
+		return conanArtifact(t, db, "pkg:conan/zlib", conanZlibPackage.filename)
+	}
+
+	conanGet(t, srv, conanZlibPackage)
+	conanGet(t, srv, conanZlibPackage)
+	if got := downloads.Load(); got != 1 {
+		t.Fatalf("upstream downloads = %d, want 1", got)
+	}
+	if !cached().LastAccessedAt.Valid {
+		t.Fatal("cache hit did not record an access")
+	}
+
+	rules := retention.Rules{Ecosystems: map[string]time.Duration{"conan": retentionDay}}
+	now := time.Now()
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now)
+	if !cached().StoragePath.Valid {
+		t.Fatal("archive used today was evicted")
+	}
+
+	old := now.Add(-2 * retentionDay)
+	conanBackdateAll(t, db, old, old)
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, now)
+	if cached().StoragePath.Valid {
+		t.Fatal("archive unused for two days kept under a one-day rule")
+	}
+
+	conanGet(t, srv, conanZlibPackage)
+	if got := downloads.Load(); got != 2 {
+		t.Errorf("upstream downloads = %d, want 2 after the eviction", got)
+	}
+	conanBackdateAll(t, db, cached().FetchedAt.Time, now.Add(-30*retentionDay))
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now())
+	if !cached().StoragePath.Valid {
+		t.Error("refetched archive evicted again by the next sweep")
+	}
+}
+
+// A key with a namespace parses as a conan PURL but can never match a
+// stored recipe, so it is refused at validation. A plain key is accepted.
+func TestConanRetentionRefusesNamespacedKeys(t *testing.T) {
+	cfg := config.Default()
+	cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/zlib": "0"}
+	if _, err := cfg.RetentionRules(retention.Default); err != nil {
+		t.Fatalf("RetentionRules(pkg:conan/zlib): %v", err)
+	}
+	cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/acme/zlib": "0"}
+	_, err := cfg.RetentionRules(retention.Default)
+	if err == nil || !strings.Contains(err.Error(), "does not match how conan packages are stored") {
+		t.Errorf("RetentionRules(pkg:conan/acme/zlib) error = %v, want the key refused", err)
+	}
+}
+
+// A package rule covers a recipe's sources, its binary packages and its
+// builds under another user/channel, while other recipes fall under the
+// conan rule.
+func TestConanRetentionPackageRule(t *testing.T) {
+	upstream, _ := conanRetentionUpstream(t)
+	srv, db := newConanRetentionProxy(t, upstream)
+	zlib := []conanFile{conanZlibSources, conanZlibPackage, conanZlibAcme}
+	for _, f := range append(zlib, conanOpenSSLSources) {
+		conanGet(t, srv, f)
+	}
+	old := time.Now().Add(-30 * retentionDay)
+	conanBackdateAll(t, db, old, old)
+
+	cfg := config.Default()
+	cfg.Storage.Retention.Ecosystems = map[string]string{"conan": "1d"}
+	cfg.Storage.Retention.Packages = map[string]string{"pkg:conan/zlib": "0"}
+	rules, err := cfg.RetentionRules(retention.Default)
+	if err != nil {
+		t.Fatalf("RetentionRules: %v", err)
+	}
+	sweepRetention(context.Background(), db, discardLogger(), retention.Default, rules, time.Now())
+
+	for _, f := range zlib {
+		if !conanArtifact(t, db, "pkg:conan/zlib", f.filename).StoragePath.Valid {
+			t.Errorf("%s evicted despite the never-evict rule for zlib", f.filename)
+		}
+	}
+	if conanArtifact(t, db, "pkg:conan/openssl", conanRecipeRevision+"_conan_sources.tgz").StoragePath.Valid {
+		t.Error("openssl kept past the conan rule")
+	}
+}