Feat/dynamic registry mirrors

builder/mirror/config.go

@@ -0,0 +1,90 @@
+// RAINBOND, Application Management Platform
+// Copyright (C) 2014-2026 Goodrain Co., Ltd.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package mirror
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// defaultSourceURL is the goodrain-maintained mirrors.json served through the
+// jsDelivr CDN, which is reachable from mainland-China clusters.
+const defaultSourceURL = "https://cdn.jsdelivr.net/gh/goodrain/docker-mirrors@main/mirrors.json"
+
+const (
+	defaultRefreshInterval = 6 * time.Hour
+	defaultMaxCount        = 3
+)
+
+// Config controls the dynamic mirror manager. All fields come from builder
+// environment variables; zero-configuration deployments get safe defaults.
+type Config struct {
+	// Enabled gates the whole feature (DYNAMIC_REGISTRY_MIRRORS, default true).
+	Enabled bool
+	// SourceURLs are tried in order when fetching candidates (MIRROR_SOURCE_URLS).
+	SourceURLs []string
+	// RefreshInterval is the period between refresh runs (MIRROR_REFRESH_INTERVAL).
+	RefreshInterval time.Duration
+	// MaxCount caps how many alive mirrors are kept (MIRROR_MAX_COUNT).
+	MaxCount int
+}
+
+// LoadConfig builds a Config from the given env lookup (usually os.Getenv).
+// Invalid values fall back to defaults with a warning instead of failing the
+// builder startup.
+func LoadConfig(getenv func(string) string) Config {
+	cfg := Config{
+		Enabled:         true,
+		SourceURLs:      []string{defaultSourceURL},
+		RefreshInterval: defaultRefreshInterval,
+		MaxCount:        defaultMaxCount,
+	}
+	if raw := getenv("DYNAMIC_REGISTRY_MIRRORS"); raw != "" {
+		cfg.Enabled = strings.EqualFold(raw, "true")
+	}
+	if raw := getenv("MIRROR_SOURCE_URLS"); raw != "" {
+		urls := make([]string, 0)
+		for _, u := range strings.Split(raw, ",") {
+			if u = strings.TrimSpace(u); u != "" {
+				urls = append(urls, u)
+			}
+		}
+		if len(urls) > 0 {
+			cfg.SourceURLs = urls
+		}
+	}
+	if raw := getenv("MIRROR_REFRESH_INTERVAL"); raw != "" {
+		if interval, err := time.ParseDuration(raw); err == nil && interval > 0 {
+			cfg.RefreshInterval = interval
+		} else {
+			logrus.Warnf("invalid MIRROR_REFRESH_INTERVAL %q, using default %v", raw, defaultRefreshInterval)
+		}
+	}
+	if raw := getenv("MIRROR_MAX_COUNT"); raw != "" {
+		if count, err := strconv.Atoi(raw); err == nil && count > 0 {
+			cfg.MaxCount = count
+		} else {
+			logrus.Warnf("invalid MIRROR_MAX_COUNT %q, using default %d", raw, defaultMaxCount)
+		}
+	}
+	return cfg
+}

builder/mirror/config_test.go

@@ -0,0 +1,58 @@
+package mirror
+
+import (
+	"testing"
+	"time"
+)
+
+// capability_id: rainbond.builder.dynamic-mirror-config
+func TestLoadConfigDefaults(t *testing.T) {
+	cfg := LoadConfig(func(string) string { return "" })
+
+	if !cfg.Enabled {
+		t.Fatal("dynamic mirrors should default to enabled")
+	}
+	assertStringSlice(t, cfg.SourceURLs, []string{defaultSourceURL})
+	if cfg.RefreshInterval != 6*time.Hour {
+		t.Fatalf("default refresh interval = %v, want 6h", cfg.RefreshInterval)
+	}
+	if cfg.MaxCount != 3 {
+		t.Fatalf("default max count = %d, want 3", cfg.MaxCount)
+	}
+}
+
+func TestLoadConfigOverrides(t *testing.T) {
+	env := map[string]string{
+		"DYNAMIC_REGISTRY_MIRRORS": "false",
+		"MIRROR_SOURCE_URLS":       "https://a.example.com/m.json, https://b.example.com/m.json,",
+		"MIRROR_REFRESH_INTERVAL":  "30m",
+		"MIRROR_MAX_COUNT":         "5",
+	}
+	cfg := LoadConfig(func(k string) string { return env[k] })
+
+	if cfg.Enabled {
+		t.Fatal("DYNAMIC_REGISTRY_MIRRORS=false should disable")
+	}
+	assertStringSlice(t, cfg.SourceURLs, []string{"https://a.example.com/m.json", "https://b.example.com/m.json"})
+	if cfg.RefreshInterval != 30*time.Minute {
+		t.Fatalf("refresh interval = %v, want 30m", cfg.RefreshInterval)
+	}
+	if cfg.MaxCount != 5 {
+		t.Fatalf("max count = %d, want 5", cfg.MaxCount)
+	}
+}
+
+func TestLoadConfigInvalidValuesFallBack(t *testing.T) {
+	env := map[string]string{
+		"MIRROR_REFRESH_INTERVAL": "soon",
+		"MIRROR_MAX_COUNT":        "-1",
+	}
+	cfg := LoadConfig(func(k string) string { return env[k] })
+
+	if cfg.RefreshInterval != 6*time.Hour {
+		t.Fatalf("invalid interval should fall back to 6h, got %v", cfg.RefreshInterval)
+	}
+	if cfg.MaxCount != 3 {
+		t.Fatalf("invalid max count should fall back to 3, got %d", cfg.MaxCount)
+	}
+}

builder/mirror/fetcher.go

@@ -0,0 +1,124 @@
+// RAINBOND, Application Management Platform
+// Copyright (C) 2014-2026 Goodrain Co., Ltd.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version. For any non-GPL usage of Rainbond,
+// one or multiple Commercial Licenses authorized by Goodrain Co., Ltd.
+// must be obtained first.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mirror maintains a dynamically refreshed list of docker.io registry
+// mirrors. Candidates come from a remote JSON source, are filtered by a live
+// /v2/ probe and exposed to the build paths via Manager.
+package mirror
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// sourceSchemaVersion is the only mirrors.json schema version this builder
+// understands. Any other version is treated as a fetch failure so the last
+// known good list keeps being used.
+const sourceSchemaVersion = 1
+
+// maxSourceBodySize bounds the JSON source payload to protect against a
+// misconfigured URL pointing at a huge file.
+const maxSourceBodySize = 1 << 20 // 1 MiB
+
+type sourceDocument struct {
+	Version   int            `json:"version"`
+	UpdatedAt string         `json:"updated_at"`
+	Mirrors   []sourceMirror `json:"mirrors"`
+}
+
+type sourceMirror struct {
+	URL  string `json:"url"`
+	Note string `json:"note"`
+}
+
+// FetchCandidates downloads the mirror candidate list, trying each source URL
+// in order until one yields a valid document. The returned URLs keep their
+// scheme (http:// entries stay plain HTTP), are trimmed and deduplicated in
+// document order. An error is returned only when every source fails.
+func FetchCandidates(ctx context.Context, sourceURLs []string, timeout time.Duration) ([]string, error) {
+	if len(sourceURLs) == 0 {
+		return nil, fmt.Errorf("no mirror source url configured")
+	}
+	client := &http.Client{Timeout: timeout}
+	var lastErr error
+	for _, sourceURL := range sourceURLs {
+		candidates, err := fetchOneSource(ctx, client, sourceURL)
+		if err != nil {
+			logrus.Warnf("fetch mirror candidates from %s failure: %v", sourceURL, err)
+			lastErr = err
+			continue
+		}
+		return candidates, nil
+	}
+	return nil, fmt.Errorf("all mirror sources failed: %w", lastErr)
+}
+
+func fetchOneSource(ctx context.Context, client *http.Client, sourceURL string) ([]string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("build request: %w", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("request source: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status %s", resp.Status)
+	}
+	body, err := io.ReadAll(io.LimitReader(resp.Body, maxSourceBodySize))
+	if err != nil {
+		return nil, fmt.Errorf("read body: %w", err)
+	}
+	var doc sourceDocument
+	if err := json.Unmarshal(body, &doc); err != nil {
+		return nil, fmt.Errorf("parse mirrors json: %w", err)
+	}
+	if doc.Version != sourceSchemaVersion {
+		return nil, fmt.Errorf("unsupported mirrors schema version %d", doc.Version)
+	}
+	candidates := dedupeMirrorURLs(doc.Mirrors)
+	if len(candidates) == 0 {
+		return nil, fmt.Errorf("mirrors json contains no usable url")
+	}
+	return candidates, nil
+}
+
+func dedupeMirrorURLs(mirrors []sourceMirror) []string {
+	seen := make(map[string]struct{}, len(mirrors))
+	result := make([]string, 0, len(mirrors))
+	for _, m := range mirrors {
+		u := strings.TrimSpace(m.URL)
+		if u == "" {
+			continue
+		}
+		if _, ok := seen[u]; ok {
+			continue
+		}
+		seen[u] = struct{}{}
+		result = append(result, u)
+	}
+	return result
+}