diff --git a/README.md b/README.md index f27e2f7..29e333b 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ | HuggingFace 加速 | 代理 `https://huggingface.co` 模型文件下载 | | Git Clone 加速 | 代理 GitHub / GitLab 的 clone、archive、raw 请求 | | GitHub API 加速 | 代理 `https://api.github.com`（解决国内 GitHub API 超时） | +| GitHub Release 下载 | 在面板持久化 Release 源，通过 DevBox 中转下载最新稳定版本中的指定资产 | | Docker v2 Auth | Token 认证代理，让 `docker pull` 不依赖直接访问上游 | | 镜像搜索 | Dashboard 搜索 npm、Docker Hub、PyPI、Conda、RubyGems、Cargo、NuGet 包 | | IP 限流 | 滚动时间窗口限流防滥用，白名单免限速，黑名单直接拒绝 | @@ -375,8 +376,19 @@ Dashboard 采用极客轻量控制台风格，面向开发者高效扫读： - Mirrors：镜像启停、上游地址修改、缓存 TTL 查看 - Git Proxy：GitHub / GitLab clone、archive、raw 命令生成与复制 - Search：npm、Docker Hub、PyPI 搜索与安装命令复制 +- Releases：保存 GitHub Releases 地址与固定资产名，一键通过 DevBox 下载最新稳定版本中的资产 - Settings：版本/运行信息、IP 限流白名单/黑名单配置 +### GitHub Release 下载 + +在 `Releases` 页面添加下载源，需要填写： + +- 名称，例如 `March7thAssistant` +- GitHub Releases 地址，例如 `https://github.com/moesnow/March7thAssistant/releases` +- 最新稳定版本中固定的资产文件名，例如 `update.7z` + +保存时 DevBox 会校验仓库与资产是否存在，之后点击 `download` 会重新确认最新稳定版本并把资产流式中转给浏览器。源保存在 `/data` 的 SQLite 数据库中，重启后仍然保留；当前仅支持公开仓库。 + ## 本地开发 ```bash @@ -392,7 +404,7 @@ docker build -t devbox:latest . ## 数据持久化 -容器 `/data` 目录存储 SQLite 数据库和缓存文件，建议映射到 Docker volume： +容器 `/data` 目录存储 SQLite 数据库（含 Release 下载源）和缓存文件，建议映射到 Docker volume： ```bash docker run -d -p 8080:8080 -v devbox-data:/data devbox:latest diff --git a/internal/dashboard/dashboard.go b/internal/dashboard/dashboard.go index d77f7a2..8702ad4 100644 --- a/internal/dashboard/dashboard.go +++ b/internal/dashboard/dashboard.go @@ -16,12 +16,18 @@ import ( ) type Dashboard struct { - store *store.Store - authToken string - publicURL string - rlConfig RateLimitConfigAccessor - saveConfig func() error - alertEngine *alert.Engine + store *store.Store + authToken string + publicURL string + rlConfig RateLimitConfigAccessor + saveConfig func() error + alertEngine *alert.Engine + releaseHTTP *http.Client + downloadHTTP *http.Client + releaseCacheMu sync.Mutex + releaseCache map[string]cachedRelease + ticketMu sync.Mutex + downloadTickets map[string]downloadTicket healthMu sync.RWMutex healthCache map[string]cachedHealth @@ -60,10 +66,14 @@ type cachedHealth struct { func New(st *store.Store, authToken string, publicURL string) *Dashboard { d := &Dashboard{ - store: st, - authToken: authToken, - publicURL: publicURL, - healthCache: make(map[string]cachedHealth), + store: st, + authToken: authToken, + publicURL: publicURL, + healthCache: make(map[string]cachedHealth), + releaseHTTP: &http.Client{Timeout: 15 * time.Second}, + downloadHTTP: newReleaseDownloadClient(), + releaseCache: make(map[string]cachedRelease), + downloadTickets: make(map[string]downloadTicket), } go d.backgroundHealthCheck() return d diff --git a/internal/dashboard/release.go b/internal/dashboard/release.go new file mode 100644 index 0000000..ddc1cf6 --- /dev/null +++ b/internal/dashboard/release.go @@ -0,0 +1,522 @@ +package dashboard + +import ( + "crypto/rand" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "mime" + "net" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "devbox/internal/store" +) + +const ( + githubAPIBase = "https://api.github.com" + githubAPIVersion = "2026-03-10" + releaseCacheTTL = 5 * time.Minute + downloadTicketTTL = 2 * time.Minute + maxDownloadTickets = 4096 +) + +var githubNamePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`) + +var errAssetNotFound = errors.New("asset not found in latest release") + +type githubRelease struct { + TagName string `json:"tag_name"` + HTMLURL string `json:"html_url"` + PublishedAt string `json:"published_at"` + Assets []githubAsset `json:"assets"` +} + +type githubAsset struct { + ID int64 `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` + Digest string `json:"digest"` + BrowserDownloadURL string `json:"browser_download_url"` +} + +type cachedRelease struct { + release githubRelease + expiresAt time.Time +} + +type downloadTicket struct { + sourceID int64 + owner string + repo string + asset githubAsset + tagName string + expiresAt time.Time +} + +type releaseSourceView struct { + store.ReleaseSource + RepositoryURL string `json:"repositoryUrl"` + TagName string `json:"tagName,omitempty"` + ReleaseURL string `json:"releaseUrl,omitempty"` + PublishedAt string `json:"publishedAt,omitempty"` + AssetSize int64 `json:"assetSize,omitempty"` + Digest string `json:"digest,omitempty"` + Available bool `json:"available"` + Error string `json:"error,omitempty"` +} + +func (d *Dashboard) ReleaseSourcesHandler(w http.ResponseWriter, r *http.Request) { + if !d.checkAuth(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + switch r.Method { + case http.MethodGet: + d.listReleaseSources(w, r, r.URL.Query().Get("refresh") == "1") + case http.MethodPost: + d.createReleaseSource(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (d *Dashboard) ReleaseSourceHandler(w http.ResponseWriter, r *http.Request) { + if !d.checkAuth(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + rest := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/release-sources/"), "/") + parts := strings.Split(rest, "/") + id, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil || id <= 0 { + http.Error(w, "invalid release source id", http.StatusBadRequest) + return + } + + if len(parts) == 1 && r.Method == http.MethodDelete { + deleted, err := d.store.DeleteReleaseSource(id) + if err != nil { + http.Error(w, "delete release source failed", http.StatusInternalServerError) + return + } + if !deleted { + http.Error(w, "release source not found", http.StatusNotFound) + return + } + writeJSON(w, map[string]string{"status": "ok"}) + return + } + + if len(parts) == 2 && parts[1] == "download-ticket" && r.Method == http.MethodPost { + d.createDownloadTicket(w, r, id) + return + } + http.Error(w, "not found", http.StatusNotFound) +} + +func (d *Dashboard) listReleaseSources(w http.ResponseWriter, r *http.Request, refresh bool) { + sources, err := d.store.ListReleaseSources() + if err != nil { + http.Error(w, "list release sources failed", http.StatusInternalServerError) + return + } + + views := make([]releaseSourceView, len(sources)) + type sourceGroup struct { + source store.ReleaseSource + indexes []int + } + groups := make(map[string]*sourceGroup) + for i, source := range sources { + key := strings.ToLower(source.Owner + "/" + source.Repo) + if group, ok := groups[key]; ok { + group.indexes = append(group.indexes, i) + } else { + groups[key] = &sourceGroup{source: source, indexes: []int{i}} + } + } + + semaphore := make(chan struct{}, 4) + var wg sync.WaitGroup + for _, group := range groups { + wg.Add(1) + go func(group *sourceGroup) { + defer wg.Done() + semaphore <- struct{}{} + defer func() { <-semaphore }() + release, err := d.latestRelease(r, group.source.Owner, group.source.Repo, refresh) + for _, index := range group.indexes { + errorMessage := "" + if err != nil { + errorMessage = err.Error() + } + views[index] = d.releaseSourceViewFromRelease(sources[index], release, errorMessage) + } + }(group) + } + wg.Wait() + writeJSON(w, views) +} + +func (d *Dashboard) createReleaseSource(w http.ResponseWriter, r *http.Request) { + var req struct { + Name string `json:"name"` + ReleaseURL string `json:"releaseUrl"` + AssetName string `json:"assetName"` + } + if !readJSON(r, &req) { + http.Error(w, "invalid request", http.StatusBadRequest) + return + } + + name := strings.TrimSpace(req.Name) + assetName := strings.TrimSpace(req.AssetName) + owner, repo, err := parseGitHubReleasesURL(req.ReleaseURL) + if err != nil || name == "" || len(name) > 80 || !validAssetName(assetName) { + http.Error(w, "invalid name, GitHub Releases URL, or asset name", http.StatusBadRequest) + return + } + + release, _, err := d.latestReleaseAsset(r, owner, repo, assetName) + if errors.Is(err, errAssetNotFound) { + http.Error(w, errAssetNotFound.Error(), http.StatusUnprocessableEntity) + return + } + if err != nil { + http.Error(w, "get latest release: "+err.Error(), http.StatusBadGateway) + return + } + + source, err := d.store.CreateReleaseSource(name, owner, repo, assetName) + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "unique") { + http.Error(w, "release source already exists", http.StatusConflict) + return + } + http.Error(w, "save release source failed", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + writeJSON(w, d.releaseSourceViewFromRelease(source, release, "")) +} + +func (d *Dashboard) releaseSourceViewFromRelease(source store.ReleaseSource, release githubRelease, releaseErr string) releaseSourceView { + view := releaseSourceView{ + ReleaseSource: source, + RepositoryURL: fmt.Sprintf("https://github.com/%s/%s", source.Owner, source.Repo), + TagName: release.TagName, + ReleaseURL: release.HTMLURL, + PublishedAt: release.PublishedAt, + Error: releaseErr, + } + if releaseErr != "" { + return view + } + asset, ok := findAsset(release, source.AssetName) + if !ok { + view.Error = errAssetNotFound.Error() + return view + } + view.Available = true + view.AssetSize = asset.Size + view.Digest = asset.Digest + return view +} + +func (d *Dashboard) createDownloadTicket(w http.ResponseWriter, r *http.Request, sourceID int64) { + source, err := d.store.GetReleaseSource(sourceID) + if errors.Is(err, sql.ErrNoRows) { + http.Error(w, "release source not found", http.StatusNotFound) + return + } + if err != nil { + http.Error(w, "get release source failed", http.StatusInternalServerError) + return + } + + release, asset, err := d.latestReleaseAsset(r, source.Owner, source.Repo, source.AssetName) + if errors.Is(err, errAssetNotFound) { + http.Error(w, errAssetNotFound.Error(), http.StatusUnprocessableEntity) + return + } + if err != nil { + http.Error(w, "get latest release: "+err.Error(), http.StatusBadGateway) + return + } + ticket, err := randomTicket() + if err != nil { + http.Error(w, "create download ticket failed", http.StatusInternalServerError) + return + } + + now := time.Now() + d.ticketMu.Lock() + for key, entry := range d.downloadTickets { + if now.After(entry.expiresAt) { + delete(d.downloadTickets, key) + } + } + if len(d.downloadTickets) >= maxDownloadTickets { + d.ticketMu.Unlock() + http.Error(w, "too many pending download tickets", http.StatusTooManyRequests) + return + } + d.downloadTickets[ticket] = downloadTicket{ + sourceID: source.ID, + owner: source.Owner, + repo: source.Repo, + asset: asset, + tagName: release.TagName, + expiresAt: now.Add(downloadTicketTTL), + } + d.ticketMu.Unlock() + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, map[string]string{ + "ticket": ticket, + "fileName": asset.Name, + "tagName": release.TagName, + }) +} + +func (d *Dashboard) ReleaseDownloadHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + ticketValue := r.URL.Query().Get("ticket") + d.ticketMu.Lock() + ticket, ok := d.downloadTickets[ticketValue] + if ok { + delete(d.downloadTickets, ticketValue) + } + d.ticketMu.Unlock() + if !ok || time.Now().After(ticket.expiresAt) { + http.Error(w, "invalid or expired download ticket", http.StatusUnauthorized) + return + } + + target := fmt.Sprintf("%s/repos/%s/%s/releases/assets/%d", githubAPIBase, + url.PathEscape(ticket.owner), url.PathEscape(ticket.repo), ticket.asset.ID) + upstreamReq, err := http.NewRequestWithContext(r.Context(), http.MethodGet, target, nil) + if err != nil { + http.Error(w, "create upstream request failed", http.StatusInternalServerError) + return + } + upstreamReq.Header.Set("Accept", "application/octet-stream") + setGitHubHeaders(upstreamReq) + resp, err := d.downloadHTTP.Do(upstreamReq) + if err != nil { + slog.Error("release download upstream failed", "source_id", ticket.sourceID, "error", err) + http.Error(w, "release download upstream failed", http.StatusBadGateway) + return + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + slog.Warn("release download upstream status", "source_id", ticket.sourceID, "status", resp.StatusCode) + http.Error(w, "release download upstream returned "+resp.Status, http.StatusBadGateway) + return + } + + contentType := resp.Header.Get("Content-Type") + if strings.Contains(strings.ToLower(contentType), "application/json") { + http.Error(w, "GitHub returned asset metadata instead of file content", http.StatusBadGateway) + return + } + if contentType == "" { + contentType = ticket.asset.ContentType + } + if contentType == "" { + contentType = "application/octet-stream" + } + w.Header().Set("Content-Type", contentType) + if contentLength := resp.Header.Get("Content-Length"); contentLength != "" { + w.Header().Set("Content-Length", contentLength) + } else if ticket.asset.Size > 0 { + w.Header().Set("Content-Length", strconv.FormatInt(ticket.asset.Size, 10)) + } + if etag := resp.Header.Get("ETag"); etag != "" { + w.Header().Set("ETag", etag) + } + w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": ticket.asset.Name})) + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, r.Context().Err()) { + slog.Warn("release download interrupted", "source_id", ticket.sourceID, "tag", ticket.tagName, "error", err) + } +} + +func (d *Dashboard) latestRelease(r *http.Request, owner, repo string, refresh bool) (githubRelease, error) { + key := releaseCacheKey(owner, repo) + if !refresh { + if release, ok := d.cachedLatestRelease(owner, repo); ok { + return release, nil + } + } + + target := fmt.Sprintf("%s/repos/%s/%s/releases/latest", githubAPIBase, url.PathEscape(owner), url.PathEscape(repo)) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, target, nil) + if err != nil { + return githubRelease{}, err + } + req.Header.Set("Accept", "application/vnd.github+json") + setGitHubHeaders(req) + resp, err := d.releaseHTTP.Do(req) + if err != nil { + return githubRelease{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + return githubRelease{}, fmt.Errorf("GitHub API returned %s", resp.Status) + } + var release githubRelease + if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&release); err != nil { + return githubRelease{}, fmt.Errorf("decode GitHub release: %w", err) + } + d.releaseCacheMu.Lock() + d.releaseCache[key] = cachedRelease{release: release, expiresAt: time.Now().Add(releaseCacheTTL)} + d.releaseCacheMu.Unlock() + return release, nil +} + +// latestReleaseAsset resolves the pinned asset from the cached latest release and +// only queries GitHub when the cache is cold or no longer lists the asset, so +// repeated downloads do not spend the unauthenticated API quota. +func (d *Dashboard) latestReleaseAsset(r *http.Request, owner, repo, assetName string) (githubRelease, githubAsset, error) { + if release, ok := d.cachedLatestRelease(owner, repo); ok { + if asset, found := findAsset(release, assetName); found { + return release, asset, nil + } + } + release, err := d.latestRelease(r, owner, repo, true) + if err != nil { + return githubRelease{}, githubAsset{}, err + } + asset, found := findAsset(release, assetName) + if !found { + return release, githubAsset{}, errAssetNotFound + } + return release, asset, nil +} + +func (d *Dashboard) cachedLatestRelease(owner, repo string) (githubRelease, bool) { + d.releaseCacheMu.Lock() + defer d.releaseCacheMu.Unlock() + entry, ok := d.releaseCache[releaseCacheKey(owner, repo)] + if !ok || !time.Now().Before(entry.expiresAt) { + return githubRelease{}, false + } + return entry.release, true +} + +func releaseCacheKey(owner, repo string) string { + return strings.ToLower(owner + "/" + repo) +} + +func parseGitHubReleasesURL(rawURL string) (string, string, error) { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Scheme != "https" || !strings.EqualFold(u.Hostname(), "github.com") || u.Port() != "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return "", "", errors.New("invalid GitHub Releases URL") + } + parts := strings.Split(strings.Trim(u.EscapedPath(), "/"), "/") + if len(parts) != 3 || !strings.EqualFold(parts[2], "releases") { + return "", "", errors.New("URL must end with /owner/repo/releases") + } + owner, err := url.PathUnescape(parts[0]) + if err != nil { + return "", "", err + } + repo, err := url.PathUnescape(parts[1]) + if err != nil { + return "", "", err + } + if !githubNamePattern.MatchString(owner) || !githubNamePattern.MatchString(repo) || len(owner) > 100 || len(repo) > 100 { + return "", "", errors.New("invalid GitHub owner or repository") + } + return owner, repo, nil +} + +func validAssetName(name string) bool { + if name == "" || len(name) > 255 || name != path.Base(name) || strings.ContainsAny(name, `/\\`) || name == "." || name == ".." { + return false + } + for _, char := range name { + if char < 0x20 || char == 0x7f { + return false + } + } + return true +} + +func findAsset(release githubRelease, name string) (githubAsset, bool) { + for _, asset := range release.Assets { + if asset.Name == name && asset.ID > 0 { + return asset, true + } + } + return githubAsset{}, false +} + +func randomTicket() (string, error) { + buf := make([]byte, 32) + if _, err := rand.Read(buf); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(buf), nil +} + +func setGitHubHeaders(req *http.Request) { + req.Header.Set("User-Agent", "DevBox-Release-Downloader") + req.Header.Set("X-GitHub-Api-Version", githubAPIVersion) +} + +func newReleaseDownloadClient() *http.Client { + return &http.Client{ + // No client-wide timeout: assets stream for minutes. Connection setup and + // the wait for response headers are bounded by the transport instead. + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("too many redirects") + } + if req.URL.Scheme != "https" || req.URL.User != nil || !allowedGitHubDownloadHost(req.URL.Hostname()) { + return errors.New("unsafe release download redirect") + } + req.Header.Del("Authorization") + return nil + }, + } +} + +func allowedGitHubDownloadHost(host string) bool { + host = strings.ToLower(host) + return host == "api.github.com" || host == "github.com" || strings.HasSuffix(host, ".githubusercontent.com") +} diff --git a/internal/dashboard/release_test.go b/internal/dashboard/release_test.go new file mode 100644 index 0000000..91d8f42 --- /dev/null +++ b/internal/dashboard/release_test.go @@ -0,0 +1,274 @@ +package dashboard + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "devbox/internal/store" +) + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestParseGitHubReleasesURL(t *testing.T) { + tests := []struct { + url string + valid bool + }{ + {"https://github.com/moesnow/March7thAssistant/releases", true}, + {"https://github.com/moesnow/March7thAssistant/releases/", true}, + {"http://github.com/moesnow/March7thAssistant/releases", false}, + {"https://example.com/moesnow/March7thAssistant/releases", false}, + {"https://github.com/moesnow/March7thAssistant", false}, + {"https://user@github.com/moesnow/March7thAssistant/releases", false}, + {"https://github.com/moesnow/March7thAssistant/releases?x=1", false}, + {"https://github.com/moesnow%2Fother/repo/releases", false}, + } + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + owner, repo, err := parseGitHubReleasesURL(tt.url) + if tt.valid && (err != nil || owner != "moesnow" || repo != "March7thAssistant") { + t.Fatalf("expected valid URL, owner=%q repo=%q err=%v", owner, repo, err) + } + if !tt.valid && err == nil { + t.Fatal("expected invalid URL") + } + }) + } +} + +func TestReleaseSourceCreateTicketAndDownload(t *testing.T) { + dashboard := newReleaseTestDashboard(t) + releaseJSON := `{ + "tag_name":"v1.2.3", + "html_url":"https://github.com/moesnow/March7thAssistant/releases/tag/v1.2.3", + "published_at":"2026-07-26T05:18:51Z", + "assets":[{"id":42,"name":"update.7z","size":7,"content_type":"application/x-7z-compressed","digest":"sha256:test"}] + }` + releaseCalls := 0 + dashboard.releaseHTTP = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.String() != githubAPIBase+"/repos/moesnow/March7thAssistant/releases/latest" { + t.Fatalf("unexpected release URL: %s", req.URL) + } + if req.Header.Get("X-GitHub-Api-Version") != githubAPIVersion { + t.Fatal("missing GitHub API version header") + } + releaseCalls++ + return testResponse(http.StatusOK, "application/json", releaseJSON), nil + })} + + body := `{"name":"M7A","releaseUrl":"https://github.com/moesnow/March7thAssistant/releases","assetName":"update.7z"}` + createReq := authorizedRequest(http.MethodPost, "/api/release-sources", body) + createRec := httptest.NewRecorder() + dashboard.ReleaseSourcesHandler(createRec, createReq) + if createRec.Code != http.StatusCreated { + t.Fatalf("create status=%d body=%s", createRec.Code, createRec.Body.String()) + } + if got := createRec.Header().Get("Content-Type"); !strings.Contains(got, "application/json") { + t.Fatalf("unexpected content type %q", got) + } + var source releaseSourceView + if err := json.NewDecoder(createRec.Body).Decode(&source); err != nil { + t.Fatal(err) + } + if !source.Available || source.TagName != "v1.2.3" || source.AssetSize != 7 { + t.Fatalf("unexpected source: %+v", source) + } + + ticketReq := authorizedRequest(http.MethodPost, "/api/release-sources/1/download-ticket", "") + ticketRec := httptest.NewRecorder() + dashboard.ReleaseSourceHandler(ticketRec, ticketReq) + if ticketRec.Code != http.StatusOK { + t.Fatalf("ticket status=%d body=%s", ticketRec.Code, ticketRec.Body.String()) + } + var ticketResult map[string]string + if err := json.NewDecoder(ticketRec.Body).Decode(&ticketResult); err != nil { + t.Fatal(err) + } + if ticketResult["ticket"] == "" { + t.Fatal("empty download ticket") + } + if releaseCalls != 1 { + t.Fatalf("expected the warm release cache to serve the ticket, GitHub calls=%d", releaseCalls) + } + + dashboard.downloadHTTP = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + if req.URL.String() != githubAPIBase+"/repos/moesnow/March7thAssistant/releases/assets/42" { + t.Fatalf("unexpected asset URL: %s", req.URL) + } + if req.Header.Get("Accept") != "application/octet-stream" { + t.Fatal("missing asset accept header") + } + resp := testResponse(http.StatusOK, "application/x-7z-compressed", "payload") + resp.Header.Set("Content-Length", "7") + return resp, nil + })} + downloadURL := "/api/release-download?ticket=" + ticketResult["ticket"] + downloadRec := httptest.NewRecorder() + dashboard.ReleaseDownloadHandler(downloadRec, httptest.NewRequest(http.MethodGet, downloadURL, nil)) + if downloadRec.Code != http.StatusOK || downloadRec.Body.String() != "payload" { + t.Fatalf("download status=%d body=%q", downloadRec.Code, downloadRec.Body.String()) + } + if disposition := downloadRec.Header().Get("Content-Disposition"); !strings.Contains(disposition, "update.7z") { + t.Fatalf("unexpected disposition %q", disposition) + } + + replayRec := httptest.NewRecorder() + dashboard.ReleaseDownloadHandler(replayRec, httptest.NewRequest(http.MethodGet, downloadURL, nil)) + if replayRec.Code != http.StatusUnauthorized { + t.Fatalf("replayed ticket status=%d", replayRec.Code) + } +} + +func TestReleaseHandlersRequireAuth(t *testing.T) { + dashboard := newReleaseTestDashboard(t) + rec := httptest.NewRecorder() + dashboard.ReleaseSourcesHandler(rec, httptest.NewRequest(http.MethodGet, "/api/release-sources", nil)) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status=%d", rec.Code) + } +} + +func TestLatestReleaseAssetRefreshesWhenCachedAssetMissing(t *testing.T) { + dashboard := newReleaseTestDashboard(t) + dashboard.releaseCache[releaseCacheKey("moesnow", "March7thAssistant")] = cachedRelease{ + release: githubRelease{TagName: "v1.0.0"}, + expiresAt: time.Now().Add(releaseCacheTTL), + } + calls := 0 + dashboard.releaseHTTP = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + calls++ + body := `{"tag_name":"v1.2.3","assets":[{"id":42,"name":"update.7z","size":7}]}` + return testResponse(http.StatusOK, "application/json", body), nil + })} + + req := httptest.NewRequest(http.MethodGet, "/api/release-sources", nil) + release, asset, err := dashboard.latestReleaseAsset(req, "moesnow", "March7thAssistant", "update.7z") + if err != nil || calls != 1 || release.TagName != "v1.2.3" || asset.ID != 42 { + t.Fatalf("unexpired cache without the asset was not refreshed: calls=%d release=%+v asset=%+v err=%v", calls, release, asset, err) + } + + if _, _, err := dashboard.latestReleaseAsset(req, "moesnow", "March7thAssistant", "missing.7z"); !errors.Is(err, errAssetNotFound) { + t.Fatalf("expected errAssetNotFound, got %v", err) + } +} + +func TestLatestReleaseAssetRefreshesExpiredCache(t *testing.T) { + dashboard := newReleaseTestDashboard(t) + key := releaseCacheKey("moesnow", "March7thAssistant") + // Expired entry that still lists the asset: only a TTL check forces a re-query. + dashboard.releaseCache[key] = cachedRelease{ + release: githubRelease{ + TagName: "v1.0.0", + Assets: []githubAsset{{ID: 7, Name: "update.7z", Size: 3}}, + }, + expiresAt: time.Now().Add(-time.Minute), + } + calls := 0 + dashboard.releaseHTTP = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + calls++ + body := `{"tag_name":"v1.2.3","assets":[{"id":42,"name":"update.7z","size":7}]}` + return testResponse(http.StatusOK, "application/json", body), nil + })} + + req := httptest.NewRequest(http.MethodGet, "/api/release-sources", nil) + release, asset, err := dashboard.latestReleaseAsset(req, "moesnow", "March7thAssistant", "update.7z") + if err != nil || calls != 1 || release.TagName != "v1.2.3" || asset.ID != 42 { + t.Fatalf("expired cache was not refreshed: calls=%d release=%+v asset=%+v err=%v", calls, release, asset, err) + } + if _, _, err := dashboard.latestReleaseAsset(req, "moesnow", "March7thAssistant", "update.7z"); err != nil || calls != 1 { + t.Fatalf("refreshed entry was not cached: calls=%d err=%v", calls, err) + } +} + +func TestReleaseDownloadClientTimeouts(t *testing.T) { + client := newReleaseDownloadClient() + if client.Timeout != 0 { + t.Fatalf("client timeout must stay unset for large downloads, got %s", client.Timeout) + } + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("unexpected transport %T", client.Transport) + } + if transport.ResponseHeaderTimeout <= 0 || transport.TLSHandshakeTimeout <= 0 || transport.DialContext == nil { + t.Fatalf("transport does not bound connection setup: %+v", transport) + } +} + +func TestReleaseDownloadRedirectPolicy(t *testing.T) { + client := newReleaseDownloadClient() + check := client.CheckRedirect + via := []*http.Request{{URL: mustURL(t, "https://api.github.com/start")}} + for _, target := range []string{ + "https://github.com/file", + "https://objects.githubusercontent.com/file", + "https://release-assets.githubusercontent.com/file", + } { + if err := check(&http.Request{URL: mustURL(t, target), Header: make(http.Header)}, via); err != nil { + t.Fatalf("expected allowed redirect %s: %v", target, err) + } + } + for _, target := range []string{ + "http://objects.githubusercontent.com/file", + "https://githubusercontent.com.example.com/file", + "https://example.com/file", + } { + if err := check(&http.Request{URL: mustURL(t, target), Header: make(http.Header)}, via); err == nil { + t.Fatalf("expected blocked redirect %s", target) + } + } +} + +func newReleaseTestDashboard(t *testing.T) *Dashboard { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "devbox.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + return &Dashboard{ + store: st, + authToken: "secret", + releaseCache: make(map[string]cachedRelease), + downloadTickets: make(map[string]downloadTicket), + } +} + +func authorizedRequest(method, target, body string) *http.Request { + req := httptest.NewRequest(method, target, bytes.NewBufferString(body)) + req.Header.Set("Authorization", "Bearer secret") + if body != "" { + req.Header.Set("Content-Type", "application/json") + } + return req +} + +func testResponse(status int, contentType, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Status: http.StatusText(status), + Header: http.Header{"Content-Type": []string{contentType}}, + Body: io.NopCloser(strings.NewReader(body)), + } +} + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatal(err) + } + return u +} diff --git a/internal/server/server.go b/internal/server/server.go index 4e78a79..480dfdd 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -166,6 +166,9 @@ func (s *Server) Start() error { mux.HandleFunc("/api/auth/login", s.dash.LoginHandler) mux.HandleFunc("/api/auth/check", s.dash.AuthCheckHandler) mux.HandleFunc("/api/search", s.search.Search) + mux.HandleFunc("/api/release-sources", s.dash.ReleaseSourcesHandler) + mux.HandleFunc("/api/release-sources/", s.dash.ReleaseSourceHandler) + mux.HandleFunc("/api/release-download", s.dash.ReleaseDownloadHandler) // Docker v2 registry API — proxy handles auth transparently mux.HandleFunc("/v2/", s.wrapWithDynamicStats(registryStatsName, s.registryV2Handler)) @@ -413,10 +416,10 @@ func (s *Server) proxyRegistryRequest(w http.ResponseWriter, r *http.Request, ta w.Header().Set("Www-Authenticate", wwAuth) } w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") - w.WriteHeader(401) - _, _ = io.Copy(w, resp.Body) - return -} + w.WriteHeader(401) + _, _ = io.Copy(w, resp.Body) + return + } // Copy response headers w.Header().Set("Docker-Distribution-API-Version", "registry/2.0") diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 00dbf2a..a9f7cc7 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "os" + "path/filepath" "time" _ "modernc.org/sqlite" @@ -14,7 +15,7 @@ type Store struct { } func New(path string) (*Store, error) { - if err := os.MkdirAll(path[:len(path)-len("/devbox.db")], 0755); err != nil { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return nil, fmt.Errorf("create data dir: %w", err) } // Enable WAL mode and busy timeout for better concurrency @@ -48,6 +49,15 @@ func (s *Store) initSchema() error { error_msg TEXT DEFAULT '', checked_at DATETIME DEFAULT CURRENT_TIMESTAMP ); + CREATE TABLE IF NOT EXISTS release_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + owner TEXT NOT NULL COLLATE NOCASE, + repo TEXT NOT NULL COLLATE NOCASE, + asset_name TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE(owner, repo, asset_name) + ); CREATE INDEX IF NOT EXISTS idx_traffic_mirror ON traffic(mirror); CREATE INDEX IF NOT EXISTS idx_traffic_created ON traffic(created_at); ` @@ -55,6 +65,67 @@ func (s *Store) initSchema() error { return err } +type ReleaseSource struct { + ID int64 `json:"id"` + Name string `json:"name"` + Owner string `json:"owner"` + Repo string `json:"repo"` + AssetName string `json:"assetName"` + CreatedAt string `json:"createdAt"` +} + +func (s *Store) CreateReleaseSource(name, owner, repo, assetName string) (ReleaseSource, error) { + result, err := s.db.Exec( + "INSERT INTO release_sources (name, owner, repo, asset_name, created_at) VALUES (?, ?, ?, ?, ?)", + name, owner, repo, assetName, time.Now().UTC().Format(time.RFC3339), + ) + if err != nil { + return ReleaseSource{}, err + } + id, err := result.LastInsertId() + if err != nil { + return ReleaseSource{}, err + } + return s.GetReleaseSource(id) +} + +func (s *Store) ListReleaseSources() ([]ReleaseSource, error) { + rows, err := s.db.Query( + "SELECT id, name, owner, repo, asset_name, created_at FROM release_sources ORDER BY id", + ) + if err != nil { + return nil, err + } + defer rows.Close() + + sources := make([]ReleaseSource, 0) + for rows.Next() { + var source ReleaseSource + if err := rows.Scan(&source.ID, &source.Name, &source.Owner, &source.Repo, &source.AssetName, &source.CreatedAt); err != nil { + return nil, err + } + sources = append(sources, source) + } + return sources, rows.Err() +} + +func (s *Store) GetReleaseSource(id int64) (ReleaseSource, error) { + var source ReleaseSource + err := s.db.QueryRow( + "SELECT id, name, owner, repo, asset_name, created_at FROM release_sources WHERE id = ?", id, + ).Scan(&source.ID, &source.Name, &source.Owner, &source.Repo, &source.AssetName, &source.CreatedAt) + return source, err +} + +func (s *Store) DeleteReleaseSource(id int64) (bool, error) { + result, err := s.db.Exec("DELETE FROM release_sources WHERE id = ?", id) + if err != nil { + return false, err + } + count, err := result.RowsAffected() + return count > 0, err +} + func (s *Store) RecordTraffic(mirror, method, path string, bytesIn, bytesOut, status int) error { _, err := s.db.Exec( "INSERT INTO traffic (mirror, method, path, bytes_in, bytes_out, status, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", diff --git a/internal/store/sqlite_test.go b/internal/store/sqlite_test.go new file mode 100644 index 0000000..0d9e5de --- /dev/null +++ b/internal/store/sqlite_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "path/filepath" + "testing" +) + +func TestReleaseSourceCRUDAndPersistence(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "devbox.db") + st, err := New(dbPath) + if err != nil { + t.Fatal(err) + } + + created, err := st.CreateReleaseSource("M7A", "moesnow", "March7thAssistant", "update.7z") + if err != nil { + t.Fatal(err) + } + if created.ID == 0 || created.Name != "M7A" || created.AssetName != "update.7z" { + t.Fatalf("unexpected source: %+v", created) + } + if _, err := st.CreateReleaseSource("duplicate", "moesnow", "March7thAssistant", "update.7z"); err == nil { + t.Fatal("expected duplicate source error") + } + if _, err := st.CreateReleaseSource("case duplicate", "MOESNOW", "march7thassistant", "update.7z"); err == nil { + t.Fatal("expected case-insensitive duplicate source error") + } + if err := st.Close(); err != nil { + t.Fatal(err) + } + + st, err = New(dbPath) + if err != nil { + t.Fatal(err) + } + defer st.Close() + sources, err := st.ListReleaseSources() + if err != nil { + t.Fatal(err) + } + if len(sources) != 1 || sources[0].ID != created.ID { + t.Fatalf("source was not persisted: %+v", sources) + } + deleted, err := st.DeleteReleaseSource(created.ID) + if err != nil || !deleted { + t.Fatalf("delete source: deleted=%v err=%v", deleted, err) + } + deleted, err = st.DeleteReleaseSource(created.ID) + if err != nil || deleted { + t.Fatalf("delete missing source: deleted=%v err=%v", deleted, err) + } +} diff --git a/web/src/App.vue b/web/src/App.vue index 72f4396..12bc265 100644 --- a/web/src/App.vue +++ b/web/src/App.vue @@ -6,6 +6,7 @@ const navItems = [ { to: '/mirrors', label: 'Mirrors', code: 'mir' }, { to: '/gitproxy', label: 'Git Proxy', code: 'git' }, { to: '/search', label: 'Search', code: 'find' }, + { to: '/releases', label: 'Releases', code: 'rel' }, { to: '/settings', label: 'Settings', code: 'cfg' }, ] diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 22e8a53..a6c014c 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -120,3 +120,53 @@ export async function updateRateLimitConfig(config: { }) { return api.put("/config/ratelimit", config).then((r) => r.data); } + +export interface ReleaseSource { + id: number; + name: string; + owner: string; + repo: string; + assetName: string; + createdAt: string; + repositoryUrl: string; + tagName?: string; + releaseUrl?: string; + publishedAt?: string; + assetSize?: number; + digest?: string; + available: boolean; + error?: string; +} + +export async function getReleaseSources( + refresh = false, +): Promise { + return api + .get("/release-sources", { params: refresh ? { refresh: "1" } : undefined }) + .then((r) => r.data); +} + +export async function createReleaseSource(source: { + name: string; + releaseUrl: string; + assetName: string; +}): Promise { + return api.post("/release-sources", source).then((r) => r.data); +} + +export async function deleteReleaseSource(id: number) { + return api.delete(`/release-sources/${id}`).then((r) => r.data); +} + +export async function createReleaseDownloadTicket(id: number): Promise<{ + ticket: string; + fileName: string; + tagName: string; +}> { + return api.post(`/release-sources/${id}/download-ticket`).then((r) => r.data); +} + +export function getReleaseDownloadUrl(ticket: string): string { + const base = import.meta.env.VITE_API_BASE_URL || "/api"; + return `${base.replace(/\/$/, "")}/release-download?ticket=${encodeURIComponent(ticket)}`; +} diff --git a/web/src/router/index.ts b/web/src/router/index.ts index e38ac20..cb0600a 100644 --- a/web/src/router/index.ts +++ b/web/src/router/index.ts @@ -5,6 +5,7 @@ import GitProxy from '../views/GitProxy.vue' import Settings from '../views/Settings.vue' import Login from '../views/Login.vue' import Search from '../views/Search.vue' +import Releases from '../views/Releases.vue' import { isLoggedIn, checkAuthRequired, initAuth } from '../api/client' const routes = [ @@ -13,6 +14,7 @@ const routes = [ { path: '/mirrors', component: Mirrors }, { path: '/gitproxy', component: GitProxy }, { path: '/search', component: Search }, + { path: '/releases', component: Releases }, { path: '/settings', component: Settings }, ] diff --git a/web/src/views/Releases.vue b/web/src/views/Releases.vue new file mode 100644 index 0000000..ee17f53 --- /dev/null +++ b/web/src/views/Releases.vue @@ -0,0 +1,237 @@ + + + diff --git a/web/src/views/Settings.vue b/web/src/views/Settings.vue index 59b84b4..2676734 100644 --- a/web/src/views/Settings.vue +++ b/web/src/views/Settings.vue @@ -58,6 +58,7 @@ const capabilities = [ 'Mirror Search', 'IP Rate Limiting', 'HuggingFace Proxy', + 'Release Downloads', ]