Cache conda packages per subdir

internal/handler/conda.go

@@ -113,8 +113,11 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) {
 		"channel", channel, "arch", arch, "name", name, "version", version)
 
 	upstreamURL := h.upstreamURL + r.URL.Path
+	// The same filename can hold different bytes in each subdir, so the
+	// subdir is part of the cache key.
+	cacheFilename := arch + "/" + filename
 
-	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, filename, upstreamURL)
+	result, err := h.proxy.GetOrFetchArtifactFromURL(r.Context(), "conda", packageName, version, cacheFilename, upstreamURL)
 	if err != nil {
 		h.proxy.serveArtifactError(w, err, "failed to fetch package")
 		return

internal/handler/conda_subdir_test.go

@@ -0,0 +1,61 @@
+package handler
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/git-pkgs/registries/fetch"
+)
+
+// The same conda filename can hold different bytes in each subdir, so each
+// subdir must get its own cache entry.
+func TestCondaDownloadCachesPerSubdir(t *testing.T) {
+	const filename = "python_abi-3.11-4_cp311.conda"
+	packages := map[string]string{
+		"/conda-forge/linux-64/" + filename:  "linux-64 bytes",
+		"/conda-forge/osx-arm64/" + filename: "osx-arm64 bytes",
+	}
+
+	var available atomic.Bool
+	available.Store(true)
+	var requests atomic.Int32
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		data, ok := packages[r.URL.Path]
+		if !available.Load() || !ok {
+			http.NotFound(w, r)
+			return
+		}
+		requests.Add(1)
+		_, _ = w.Write([]byte(data))
+	}))
+	defer upstream.Close()
+
+	proxy, _, _, _ := setupTestProxy(t)
+	fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0))
+	proxy.Fetcher = fetcher
+	t.Cleanup(func() { _ = fetcher.Close() })
+	routes := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes()
+
+	get := func(path string) string {
+		w := httptest.NewRecorder()
+		routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
+		if w.Code != http.StatusOK {
+			t.Fatalf("%s: status = %d: %s", path, w.Code, w.Body.String())
+		}
+		return w.Body.String()
+	}
+
+	for _, round := range []string{"fetch", "cached"} {
+		for path, want := range packages {
+			if got := get(path); got != want {
+				t.Errorf("%s %s: body = %q, want %q", round, path, got, want)
+			}
+		}
+		available.Store(false)
+	}
+	if got := requests.Load(); got != 2 {
+		t.Errorf("upstream requests = %d, want 2 (one per subdir)", got)
+	}
+}

internal/handler/download_test.go

@@ -488,7 +488,7 @@ func TestPubHandler_CacheMiss(t *testing.T) {
 
 func TestCondaHandler_DownloadCacheHit(t *testing.T) {
 	proxy, db, store, _ := setupTestProxy(t)
-	seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "numpy-1.24.0-py311h64a7726_0.conda", "conda pkg")
+	seedPackageWithPURL(t, db, store, "conda", "main/numpy", "1.24.0", "linux-64/numpy-1.24.0-py311h64a7726_0.conda", "conda pkg")
 
 	h := NewCondaHandler(proxy, "http://localhost")
 	srv := httptest.NewServer(h.Routes())
@@ -511,7 +511,7 @@ func TestCondaHandler_DownloadCacheHit(t *testing.T) {
 
 func TestCondaHandler_DownloadTarBz2CacheHit(t *testing.T) {
 	proxy, db, store, _ := setupTestProxy(t)
-	seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data")
+	seedPackageWithPURL(t, db, store, "conda", "main/scipy", "1.11.0", "linux-64/scipy-1.11.0-py311hb2e3ea1_0.tar.bz2", "tar bz2 data")
 
 	h := NewCondaHandler(proxy, "http://localhost")
 	srv := httptest.NewServer(h.Routes())

internal/server/retention_conda_test.go

@@ -69,7 +69,7 @@ func condaGet(t *testing.T, url string) {
 	}
 }
 
-// condaArtifact returns the record of the package file named filename.
+// condaArtifact returns the record cached as filename, which is {subdir}/{file}.
 func condaArtifact(t *testing.T, db *database.DB, filename string) database.Artifact {
 	t.Helper()
 	var art database.Artifact
@@ -103,7 +103,7 @@ func TestCondaRegistersForRetention(t *testing.T) {
 func TestCondaRetentionEvictsAndRefetches(t *testing.T) {
 	upstream, downloads := condaRetentionUpstream(t)
 	srv, db := newCondaRetentionProxy(t, upstream)
-	const file = "requests-2.31.0-pyhd8ed1ab_0.conda"
+	const file = "noarch/requests-2.31.0-pyhd8ed1ab_0.conda"
 
 	condaGet(t, srv.URL+condaRequestsPath)
 	condaGet(t, srv.URL+condaRequestsPath)
@@ -162,7 +162,7 @@ func TestCondaRetentionRefusesKeysWithoutChannel(t *testing.T) {
 func TestCondaRetentionPackageRule(t *testing.T) {
 	upstream, _ := condaRetentionUpstream(t)
 	srv, db := newCondaRetentionProxy(t, upstream)
-	const forgeNumpy, anacondaNumpy, requests = "numpy-1.24.0-py311h64a7726_0.conda", "numpy-1.24.0-py311h08b1b3b_0.conda", "requests-2.31.0-pyhd8ed1ab_0.conda"
+	const forgeNumpy, anacondaNumpy, requests = "linux-64/numpy-1.24.0-py311h64a7726_0.conda", "linux-64/numpy-1.24.0-py311h08b1b3b_0.conda", "noarch/requests-2.31.0-pyhd8ed1ab_0.conda"
 	condaGet(t, srv.URL+condaForgeNumpyPath)
 	condaGet(t, srv.URL+condaAnacondaNumpy)
 	condaGet(t, srv.URL+condaRequestsPath)