diff --git a/auth/gitee.go b/auth/gitee.go index ab172a7..b2eb5c5 100644 --- a/auth/gitee.go +++ b/auth/gitee.go @@ -28,11 +28,18 @@ var ( ) var ( - allowedRepos = []string{"openeuler", "src-openeuler", "lfs-org", "openeuler-test"} - uploadPermissions = []string{"admin", "developer"} - downloadPermissions = []string{"admin", "developer", "read"} + defaultAllowedRepos = []string{"openeuler", "src-openeuler", "lfs-org", "openeuler-test"} + allowedRepos = []string{"openeuler", "src-openeuler", "lfs-org", "openeuler-test"} + uploadPermissions = []string{"admin", "developer"} + downloadPermissions = []string{"admin", "developer", "read"} + namespacePlatformMap = map[string]string{} + githubDefaultToken string + githubAllowedOrgs []string ) +// supportedPlatforms enumerates the platforms recognised by ResolvePlatform. +var supportedPlatforms = []string{"gitee", "gitcode", "github"} + const ( refer = "referer" accept = "Accept" @@ -113,9 +120,77 @@ func Init(cfg *config.Config) error { } gitCodeSwitch = cfg.GitCodeSwitch + + buildNamespacePlatformMap(cfg) + githubAllowedOrgs = cfg.GithubAllowedOrgs + githubDefaultToken = cfg.GithubDefaultToken + if githubDefaultToken == "" { + githubDefaultToken = os.Getenv("GITHUB_DEFAULT_TOKEN") + } return nil } +// buildNamespacePlatformMap constructs the owner->platform lookup from config, +// keeping the hardcoded openEuler namespaces as backward-compatible defaults so +// existing deployments keep working without any configuration change. +func buildNamespacePlatformMap(cfg *config.Config) { + namespacePlatformMap = make(map[string]string) + allowedRepos = make([]string, len(defaultAllowedRepos)) + copy(allowedRepos, defaultAllowedRepos) + for _, ns := range defaultAllowedRepos { + namespacePlatformMap[ns] = "gitee" + } + for _, m := range cfg.AllowedNamespaces { + if m.Namespace == "" { + continue + } + platform := m.Platform + if !isSupportedPlatform(platform) { + platform = "gitee" + } + namespacePlatformMap[m.Namespace] = platform + // gitee/gitcode owners must also be admissible in CheckRepoOwner. + if platform != "github" { + allowedRepos = append(allowedRepos, m.Namespace) + } + } +} + +func isSupportedPlatform(p string) bool { + for _, s := range supportedPlatforms { + if s == p { + return true + } + } + return false +} + +// ResolvePlatform returns the hosting platform for a given owner namespace. +// It defaults to "gitcode" when gitCodeSwitch is on, otherwise "gitee". +func ResolvePlatform(owner string) string { + if p, ok := namespacePlatformMap[owner]; ok { + return p + } + if gitCodeSwitch { + return "gitcode" + } + return "gitee" +} + +// PlatformForMetadata maps a resolved platform to the value stored in the +// lfs_obj.platform column. It keeps the legacy "atomGit" spelling for gitcode +// so existing rows remain queryable. +func PlatformForMetadata(platform string) string { + switch platform { + case "github": + return "github" + case "gitcode": + return "atomGit" + default: + return "gitee" + } +} + func InitOpenEulerParam(cfg *config.Config) error { openEulerAccountParam.GrantType = tokenParam openEulerAccountParam.Url = cfg.OpenEulerAccountConfig.UrlPath @@ -455,12 +530,7 @@ func GetLFSMapping(userInRepo UserInRepo, pythonScriptPath ...string) (map[strin repo := userInRepo.Repo username := userInRepo.Username token := userInRepo.Token - var platform string - if gitCodeSwitch { - platform = "gitcode" - } else { - platform = "gitee" - } + platform := ResolvePlatform(owner) // 确定Python脚本路径 scriptPath, err := resolveScriptPath(pythonScriptPath...) diff --git a/auth/github.go b/auth/github.go index 120eb82..80c5fde 100644 --- a/auth/github.go +++ b/auth/github.go @@ -5,12 +5,60 @@ import ( "errors" "fmt" "net/http" + "strings" + + "github.com/sirupsen/logrus" +) + +var ( + githubUploadPermissions = []string{"admin", "maintain", "write"} + githubDownloadPermissions = []string{"admin", "maintain", "write", "triage", "read"} + githubDeletePermissions = []string{"admin"} ) type ghOrg struct { Login string `json:"login"` } +// githubRepo mirrors the subset of the GitHub /repos/{owner}/{repo} response +// that BigFiles needs for ownership validation. +type githubRepo struct { + FullName string `json:"full_name"` + Owner ghOrg `json:"owner"` + Parent parent `json:"parent"` +} + +// githubPermission mirrors the GitHub collaborator permission response. +type githubPermission struct { + Permission string `json:"permission"` +} + +const ( + githubAccept = "application/vnd.github+json" +) + +// githubAPIBase is the GitHub REST API root. It is a var so unit tests can +// redirect it to an httptest server. +var githubAPIBase = "https://api.github.com" + +// githubTokenFor returns the token to use for a GitHub API call: the user PAT +// when present, otherwise the server-side default bot token. +func githubTokenFor(u UserInRepo) string { + if u.Token != "" { + return u.Token + } + return githubDefaultToken +} + +func githubHeaders(token string) http.Header { + h := http.Header{ + contentType: []string{headerContentType}, + accept: []string{githubAccept}, + authorization: []string{"Bearer " + token}, + } + return h +} + func GithubOrg(org string) func(string, string) error { return func(_, ghToken string) error { req, err := http.NewRequest("GET", "https://api.github.com/user/orgs", nil) @@ -47,3 +95,116 @@ func Static(u, p string) func(string, string) error { return nil } } + +// CheckGithubRepoOwner validates that {owner}/{repo} exists on GitHub and that +// the repository owner is an allowed GitHub organization. +func CheckGithubRepoOwner(u UserInRepo) (Repo, error) { + token := githubTokenFor(u) + path := fmt.Sprintf("%s/repos/%s/%s", githubAPIBase, u.Owner, u.Repo) + repo := new(githubRepo) + if err := getParsedResponse("GET", path, githubHeaders(token), nil, repo); err != nil { + return Repo{}, err + } + for _, allowed := range githubAllowedOrgs { + if repo.Owner.Login == allowed { + return Repo{Fullname: repo.FullName, Parent: repo.Parent}, nil + } + } + // a fork under an allowed organisation is also admissible + if repo.Parent.Fullname != "" { + parentOwner := strings.Split(repo.Parent.Fullname, "/")[0] + for _, allowed := range githubAllowedOrgs { + if parentOwner == allowed { + return Repo{Fullname: repo.FullName, Parent: repo.Parent}, nil + } + } + } + msg := "forbidden: repo has no permission to use this lfs server" + logrus.Error(fmt.Sprintf("CheckGithubRepoOwner | %s | owner:%s repo:%s", msg, u.Owner, u.Repo)) + return Repo{Fullname: repo.FullName, Parent: repo.Parent}, errors.New(msg) +} + +// VerifyGithubUser checks the collaborator permission of username on the repo +// against the operation being performed. +func VerifyGithubUser(u UserInRepo) error { + token := githubTokenFor(u) + path := fmt.Sprintf("%s/repos/%s/%s/collaborators/%s/permission", + githubAPIBase, u.Owner, u.Repo, u.Username) + perm := new(githubPermission) + if err := getParsedResponse("GET", path, githubHeaders(token), nil, perm); err != nil { + msg := err.Error() + ": verify github user permission failed" + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) + } + + switch u.Operation { + case "upload": + return verifyGithubUserUpload(perm, u) + case "download": + return verifyGithubUserDownload(perm, u) + case "delete": + return verifyGithubUserDelete(perm, u) + default: + msg := "system_error: unknown operation" + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) + } +} + +func verifyGithubUserUpload(perm *githubPermission, u UserInRepo) error { + for _, p := range githubUploadPermissions { + if perm.Permission == p { + return nil + } + } + msg := fmt.Sprintf("forbidden: user %s has no permission to upload to %s/%s", + u.Username, u.Owner, u.Repo) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) +} + +func verifyGithubUserDownload(perm *githubPermission, u UserInRepo) error { + for _, p := range githubDownloadPermissions { + if perm.Permission == p { + return nil + } + } + msg := fmt.Sprintf("forbidden: user %s has no permission to download from %s/%s", + u.Username, u.Owner, u.Repo) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) +} + +func verifyGithubUserDelete(perm *githubPermission, u UserInRepo) error { + for _, p := range githubDeletePermissions { + if perm.Permission == p { + return nil + } + } + msg := fmt.Sprintf("forbidden: user %s has no permission to delete in %s/%s", + u.Username, u.Owner, u.Repo) + logrus.Error(fmt.Sprintf(formatLogString, verifyLog, msg)) + return errors.New(msg) +} + +// GithubAuth mirrors GiteeAuth: first validate repo ownership, then the user's +// collaborator permission. For GitHub the PAT is passed verbatim in Basic Auth +// (password field), no OAuth token exchange is performed. +func GithubAuth() func(UserInRepo) error { + return func(userInRepo UserInRepo) error { + userInRepo.Token = userInRepo.Password + if _, err := CheckGithubRepoOwner(userInRepo); err != nil { + return err + } + return VerifyGithubUser(userInRepo) + } +} + +// CheckRepoOwnerByPlatform dispatches repo-ownership validation to the +// platform-specific implementation. +func CheckRepoOwnerByPlatform(platform string, u UserInRepo) (Repo, error) { + if platform == "github" { + return CheckGithubRepoOwner(u) + } + return CheckRepoOwner(u) +} diff --git a/auth/github_test.go b/auth/github_test.go index f325e54..12f1f58 100644 --- a/auth/github_test.go +++ b/auth/github_test.go @@ -1,8 +1,13 @@ package auth import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "testing" + "github.com/metalogical/BigFiles/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -41,3 +46,457 @@ func (s *SuiteGithub) TestGithubOrg() { func TestGithub(t *testing.T) { suite.Run(t, new(SuiteGithub)) } + +// -------------------- GitHub platform auth tests (httptest) -------------------- + +// ghMockServer builds an httptest server that stubs the GitHub REST endpoints +// used by BigFiles. repoOwner is the login returned by /repos/{o}/{r}; +// permission is returned by the collaborator endpoint. statusRepo/statusPerm +// override the HTTP status codes. +type ghMock struct { + repoOwner string + repoParent string + permission string + statusRepo int + statusPerm int + repoCalled bool + permCalled bool + permToken string +} + +func newGHMockServer(m *ghMock) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/repos/", func(w http.ResponseWriter, r *http.Request) { + if isCollaboratorPath(r.URL.Path) { + m.permCalled = true + m.permToken = r.Header.Get("Authorization") + if m.statusPerm != 0 { + w.WriteHeader(m.statusPerm) + return + } + _ = json.NewEncoder(w).Encode(map[string]string{"permission": m.permission}) + return + } + m.repoCalled = true + if m.statusRepo != 0 { + w.WriteHeader(m.statusRepo) + return + } + owner := m.repoOwner + if owner == "" { + owner = "mindspore-ai" + } + resp := map[string]interface{}{ + "full_name": fmt.Sprintf("%s/%s", owner, "repo"), + "owner": map[string]string{"login": owner}, + } + if m.repoParent != "" { + resp["parent"] = map[string]string{"full_name": m.repoParent} + } + _ = json.NewEncoder(w).Encode(resp) + }) + return httptest.NewServer(mux) +} + +// isCollaboratorPath reports whether the path targets the collaborator +// permission endpoint rather than the repo metadata endpoint. +func isCollaboratorPath(path string) bool { + return indexOf(path, "/collaborators/") >= 0 && indexOf(path, "/permission") >= 0 +} + +func indexOf(s, sub string) int { + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + return i + } + } + return -1 +} + +func TestResolvePlatform(t *testing.T) { + origMap := namespacePlatformMap + origSwitch := gitCodeSwitch + defer func() { + namespacePlatformMap = origMap + gitCodeSwitch = origSwitch + }() + + namespacePlatformMap = map[string]string{ + "mindspore-ai": "github", + "openeuler": "gitee", + } + gitCodeSwitch = false + + assert.Equal(t, "github", ResolvePlatform("mindspore-ai")) + assert.Equal(t, "gitee", ResolvePlatform("openeuler")) + // unconfigured owner falls back to gitee + assert.Equal(t, "gitee", ResolvePlatform("someone-else")) + + // with gitCodeSwitch on, unconfigured owner resolves to gitcode + gitCodeSwitch = true + assert.Equal(t, "gitcode", ResolvePlatform("someone-else")) + // but configured github owner still wins + assert.Equal(t, "github", ResolvePlatform("mindspore-ai")) +} + +func TestPlatformForMetadata(t *testing.T) { + assert.Equal(t, "github", PlatformForMetadata("github")) + assert.Equal(t, "atomGit", PlatformForMetadata("gitcode")) + assert.Equal(t, "gitee", PlatformForMetadata("gitee")) + assert.Equal(t, "gitee", PlatformForMetadata("unknown")) +} + +func TestIsSupportedPlatform(t *testing.T) { + assert.True(t, isSupportedPlatform("gitee")) + assert.True(t, isSupportedPlatform("gitcode")) + assert.True(t, isSupportedPlatform("github")) + assert.False(t, isSupportedPlatform("gitlab")) + assert.False(t, isSupportedPlatform("")) +} + +func TestBuildNamespacePlatformMap(t *testing.T) { + origMap := namespacePlatformMap + origRepos := allowedRepos + defer func() { + namespacePlatformMap = origMap + allowedRepos = origRepos + }() + + cfg := &config.Config{ + AllowedNamespaces: []config.NamespaceMapping{ + {Namespace: "mindspore-ai", Platform: "github"}, + {Namespace: "new-gitee-org", Platform: "gitee"}, + {Namespace: "bad-platform-org", Platform: "gitlab"}, // unsupported -> gitee + {Namespace: "", Platform: "gitee"}, // empty -> skipped + }, + } + buildNamespacePlatformMap(cfg) + + // configured github namespace + assert.Equal(t, "github", namespacePlatformMap["mindspore-ai"]) + // configured gitee namespace + assert.Equal(t, "gitee", namespacePlatformMap["new-gitee-org"]) + // default namespaces preserved + assert.Equal(t, "gitee", namespacePlatformMap["openeuler"]) + assert.Equal(t, "gitee", namespacePlatformMap["src-openeuler"]) + // unsupported platform falls back to gitee + assert.Equal(t, "gitee", namespacePlatformMap["bad-platform-org"]) + // gitee/gitcode namespaces are admissible in CheckRepoOwner (allowedRepos) + assert.Contains(t, allowedRepos, "new-gitee-org") + assert.Contains(t, allowedRepos, "bad-platform-org") + // github namespace is NOT added to allowedRepos (handled by githubAllowedOrgs) + for _, r := range allowedRepos { + assert.NotEqual(t, "mindspore-ai", r) + } +} + +func TestCheckGithubRepoOwner(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{repoOwner: "mindspore-ai"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + u := UserInRepo{Owner: "mindspore-ai", Repo: "repo", Token: "pat"} + repo, err := CheckGithubRepoOwner(u) + assert.NoError(t, err) + assert.Equal(t, "mindspore-ai/repo", repo.Fullname) + assert.True(t, m.repoCalled) +} + +func TestCheckGithubRepoOwner_DisallowedOrg(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{repoOwner: "evil-org"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + _, err := CheckGithubRepoOwner(UserInRepo{Owner: "evil-org", Repo: "repo"}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "forbidden") +} + +func TestCheckGithubRepoOwner_ForkOfAllowedOrg(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{repoOwner: "personal-fork", repoParent: "mindspore-ai/repo"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + repo, err := CheckGithubRepoOwner(UserInRepo{Owner: "personal-fork", Repo: "repo"}) + assert.NoError(t, err) + assert.NotEmpty(t, repo.Parent.Fullname) +} + +func TestCheckGithubRepoOwner_NotFound(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{statusRepo: http.StatusNotFound} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + _, err := CheckGithubRepoOwner(UserInRepo{Owner: "mindspore-ai", Repo: "repo"}) + assert.Error(t, err) +} + +func TestCheckGithubRepoOwner_UsesDefaultToken(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + origDefault := githubDefaultToken + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + githubDefaultToken = origDefault + }() + githubAllowedOrgs = []string{"mindspore-ai"} + githubDefaultToken = "server-bot-token" + + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + _ = json.NewEncoder(w).Encode(githubRepo{ + FullName: "mindspore-ai/repo", + Owner: ghOrg{Login: "mindspore-ai"}, + }) + })) + defer srv.Close() + githubAPIBase = srv.URL + + // no user token -> server default token used + _, err := CheckGithubRepoOwner(UserInRepo{Owner: "mindspore-ai", Repo: "repo"}) + assert.NoError(t, err) + assert.Equal(t, "Bearer server-bot-token", gotAuth) +} + +func TestVerifyGithubUser_Upload(t *testing.T) { + origBase := githubAPIBase + defer func() { githubAPIBase = origBase }() + + cases := []struct { + name string + permission string + op string + wantErr bool + }{ + {"write can upload", "write", "upload", false}, + {"maintain can upload", "maintain", "upload", false}, + {"admin can upload", "admin", "upload", false}, + {"read cannot upload", "read", "upload", true}, + {"triage cannot upload", "triage", "upload", true}, + {"none cannot upload", "none", "upload", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + m := &ghMock{permission: c.permission} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + err := VerifyGithubUser(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Operation: c.op, Token: "pat", + }) + if c.wantErr { + assert.Error(t, err) + assert.Contains(t, err.Error(), "forbidden") + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestVerifyGithubUser_Download(t *testing.T) { + origBase := githubAPIBase + defer func() { githubAPIBase = origBase }() + + cases := []struct { + name string + permission string + wantErr bool + }{ + {"read can download", "read", false}, + {"triage can download", "triage", false}, + {"write can download", "write", false}, + {"none cannot download", "none", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + m := &ghMock{permission: c.permission} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + err := VerifyGithubUser(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Operation: "download", Token: "pat", + }) + if c.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestVerifyGithubUser_Delete(t *testing.T) { + origBase := githubAPIBase + defer func() { githubAPIBase = origBase }() + + cases := []struct { + name string + permission string + wantErr bool + }{ + {"admin can delete", "admin", false}, + {"write cannot delete", "write", true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + m := &ghMock{permission: c.permission} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + err := VerifyGithubUser(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Operation: "delete", Token: "pat", + }) + if c.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestVerifyGithubUser_Unauthorized(t *testing.T) { + origBase := githubAPIBase + defer func() { githubAPIBase = origBase }() + + m := &ghMock{statusPerm: http.StatusUnauthorized} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + err := VerifyGithubUser(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Operation: "download", Token: "bad-pat", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unauthorized") +} + +func TestVerifyGithubUser_UnknownOperation(t *testing.T) { + origBase := githubAPIBase + defer func() { githubAPIBase = origBase }() + + m := &ghMock{permission: "write"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + err := VerifyGithubUser(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Operation: "weird", Token: "pat", + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unknown operation") +} + +func TestGithubAuth_Success(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{repoOwner: "mindspore-ai", permission: "write"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + fn := GithubAuth() + err := fn(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Password: "pat", Operation: "upload", + }) + assert.NoError(t, err) +} + +func TestGithubAuth_BadPAT(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + }() + githubAllowedOrgs = []string{"mindspore-ai"} + + m := &ghMock{statusRepo: http.StatusUnauthorized} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + fn := GithubAuth() + err := fn(UserInRepo{ + Owner: "mindspore-ai", Repo: "repo", + Username: "alice", Password: "bad-pat", Operation: "download", + }) + assert.Error(t, err) +} + +func TestCheckRepoOwnerByPlatform_Dispatch(t *testing.T) { + origBase := githubAPIBase + origOrgs := githubAllowedOrgs + origMap := namespacePlatformMap + defer func() { + githubAPIBase = origBase + githubAllowedOrgs = origOrgs + namespacePlatformMap = origMap + }() + githubAllowedOrgs = []string{"mindspore-ai"} + namespacePlatformMap = map[string]string{"mindspore-ai": "github"} + + m := &ghMock{repoOwner: "mindspore-ai"} + srv := newGHMockServer(m) + defer srv.Close() + githubAPIBase = srv.URL + + // github platform dispatches to CheckGithubRepoOwner + _, err := CheckRepoOwnerByPlatform("github", UserInRepo{Owner: "mindspore-ai", Repo: "repo", Token: "pat"}) + assert.NoError(t, err) + assert.True(t, m.repoCalled) +} diff --git a/config.example.yml b/config.example.yml index c48641f..a333db7 100644 --- a/config.example.yml +++ b/config.example.yml @@ -7,6 +7,12 @@ CLIENT_ID: "***********" CLIENT_SECRET: "***********" PATH_PREFIX: "***********" DEFAULT_TOKEN: "***********" +GITHUB_DEFAULT_TOKEN: "***********" +GITHUB_ALLOWED_ORGS: + - "mindspore-ai" +ALLOWED_NAMESPACES: + - NAMESPACE: "mindspore-ai" + PLATFORM: "github" VALIDATE_REGEXP: OWNER_REGEXP: "^[a-zA-Z]([-_.]?[a-zA-Z0-9]+)*$" REPONAME_REGEXP: "^[a-zA-Z0-9_.-]{1,189}[a-zA-Z0-9]$" diff --git a/config/config.go b/config/config.go index 035e279..c71b3b0 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,9 @@ type Config struct { ObsRegion string `json:"OBS_REGION"` DefaultToken string `json:"DEFAULT_TOKEN"` DefaultGitCodeToken string `json:"DEFAULT_GIT_CODE_TOKEN"` + GithubDefaultToken string `json:"GITHUB_DEFAULT_TOKEN"` + GithubAllowedOrgs []string `json:"GITHUB_ALLOWED_ORGS"` + AllowedNamespaces []NamespaceMapping `json:"ALLOWED_NAMESPACES"` ValidateConfig ValidateConfig `json:"VALIDATE_REGEXP"` ObsAccessKeyId string `json:"OBS_ACCESS_KEY_ID"` ObsSecretAccessKey string `json:"OBS_SECRET_ACCESS_KEY"` @@ -25,6 +28,12 @@ type Config struct { GitCodeSwitch bool `json:"GIT_CODE_SWITCH" default:"false"` } +// NamespaceMapping maps a repository owner namespace to its hosting platform. +type NamespaceMapping struct { + Namespace string `json:"NAMESPACE"` + Platform string `json:"PLATFORM"` +} + type ValidateConfig struct { OwnerRegexp string `json:"OWNER_REGEXP" required:"true"` RepoNameRegexp string `json:"REPONAME_REGEXP" required:"true"` diff --git a/docs/BasicGuide.md b/docs/BasicGuide.md index 0a7231a..3cce8b8 100644 --- a/docs/BasicGuide.md +++ b/docs/BasicGuide.md @@ -6,8 +6,29 @@ 第三方LFS服务是基于Git LFS插件,实现将Gitee仓库内的大文件上传至第三方LFS服务中的功能。目前该插件仅支持openeuler、src-openeuler组织下的仓库。 +> 自建 LFS 服务同样支持 **GitHub** 平台(如 `mindspore-ai` 组织下的仓库)。GitHub 仓库使用 **Personal Access Token(PAT)** 进行鉴权:在 `git` 操作的 Basic Auth 中,用户名填 GitHub 账号,密码填 PAT(需 `repo` 或 `Contents: read/write` 权限)。服务端会校验「仓库归属(owner 是否在允许的 GitHub 组织清单内)」与「协作者权限」,通过后签发 OBS 上传/下载预签名 URL。配置方法见下方 [GitHub 平台配置](#github-平台配置)。 + *** +## GitHub 平台配置 + +GitHub 平台支持通过配置文件开启,需在服务端 `config` 中补充以下字段(也可通过同名环境变量注入): + +| 字段 | 环境变量 | 说明 | +| --- | --- | --- | +| `GITHUB_DEFAULT_TOKEN` | `GITHUB_DEFAULT_TOKEN` | 服务端 bot PAT,用于仓库元信息查询与 OID 文件名回填兜底 | +| `GITHUB_ALLOWED_ORGS` | - | 允许使用本 LFS 服务的 GitHub 组织清单,如 `mindspore-ai` | +| `ALLOWED_NAMESPACES` | - | owner→平台映射,新增 GitHub 组织时在此登记,如 `{NAMESPACE: mindspore-ai, PLATFORM: github}` | + +GitHub 仓库侧只需在仓库根目录创建 `.lfsconfig` 指向自建 LFS: + +``` +[lfs] + url = https://artlfs.openeuler.openatom.cn/{owner}/{repo} +``` + +其中 `{owner}/{repo}` 为 GitHub 上的仓库路径(如 `mindspore-ai/repo`)。之后 `git lfs push` / `git lfs pull` 时按提示输入 GitHub 用户名与 PAT 即可。 + ## 在已有仓库启用LFS ### 下载 diff --git a/go.mod b/go.mod index 832ad2d..e4c47f0 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( bou.ke/monkey v1.0.2 github.com/go-chi/chi v4.1.2+incompatible gorm.io/driver/mysql v1.6.0 + gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 sigs.k8s.io/yaml v1.6.0 ) @@ -17,6 +18,7 @@ require ( github.com/go-sql-driver/mysql v1.9.3 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect ) diff --git a/go.sum b/go.sum index bf6ade0..18ee798 100644 --- a/go.sum +++ b/go.sum @@ -7,46 +7,35 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible h1:XQVXdk+WAJ4fSNB6mMRuYNvFWou7BZs6SZB925hPrnk= -github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.9+incompatible h1:T9+wBrjfJUrWKppRwXhDNjf6vAJy7DfZYWgkjNbxkIU= github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.9+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.3 h1:bXOww4E/J3f66rav3pX3m8w6jDE4knZjGOw8b5Y6iNE= go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -54,11 +43,10 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= diff --git a/main.go b/main.go index f2a6530..fdbdc78 100644 --- a/main.go +++ b/main.go @@ -122,13 +122,18 @@ func main() { initConfig(cfg) s, err := server.New(server.Options{ - Prefix: cfg.Prefix, - Bucket: cfg.LfsBucket, - Endpoint: cfg.ObsRegion, - CdnDomain: cfg.CdnDomain, - AccessKeyID: cfg.ObsAccessKeyId, - S3Accelerate: true, - IsAuthorized: auth.GiteeAuth(), + Prefix: cfg.Prefix, + Bucket: cfg.LfsBucket, + Endpoint: cfg.ObsRegion, + CdnDomain: cfg.CdnDomain, + AccessKeyID: cfg.ObsAccessKeyId, + S3Accelerate: true, + IsAuthorized: auth.GiteeAuth(), + IsAuthorizedByPlatform: map[string]func(auth.UserInRepo) error{ + "gitee": auth.GiteeAuth(), + "gitcode": auth.GiteeAuth(), + "github": auth.GithubAuth(), + }, SecretAccessKey: cfg.ObsSecretAccessKey, }) diff --git a/scripts/lfsNameQuery.py b/scripts/lfsNameQuery.py index 62a2628..8721c02 100644 --- a/scripts/lfsNameQuery.py +++ b/scripts/lfsNameQuery.py @@ -14,6 +14,10 @@ "gitcode": { "domain": "gitcode.com", "auth_method": "token" # GitCode必须使用Token认证[6](@ref) + }, + "github": { + "domain": "github.com", + "auth_method": "token" # GitHub Personal Access Token } } @@ -178,7 +182,7 @@ def main(platform, owner, repo, output_file="lfs_mapping.json", username=None, t if __name__ == "__main__": if len(sys.argv) < 4: print("用法: python lfsNameQuery.py [output_file] [username] [token]") - print("平台支持: gitee, gitcode") + print("平台支持: gitee, gitcode, github") sys.exit(1) args = { diff --git a/server/listfiles_test.go b/server/listfiles_test.go new file mode 100644 index 0000000..bf5bef2 --- /dev/null +++ b/server/listfiles_test.go @@ -0,0 +1,124 @@ +package server + +import ( + "testing" + + "github.com/metalogical/BigFiles/db" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// newSQLiteTestDB spins up an in-memory SQLite database, auto-migrates the +// lfs_obj table, and wires it as the global db.Db for the duration of a test. +// It returns a teardown func restoring the previous global handle. +// +// These tests need a real SQL engine because getLfsFiles/countLfsFiles build +// their WHERE clauses through gorm's chainable API; verifying the platform +// filter branch means asserting on actual query results rather than a mocked +// gorm stub (the previous Test_server_List only stubbed the functions and left +// the real SQL generation at 0% coverage). +func newSQLiteTestDB(t *testing.T) func() { + t.Helper() + gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + // :memory: databases are per-connection; pin a single connection so the + // migrated schema and inserted rows stay visible across queries. + sqlDB, err := gormDB.DB() + require.NoError(t, err) + sqlDB.SetMaxOpenConns(1) + + // Create the lfs_objs table with SQLite-compatible DDL. We cannot use + // gorm.AutoMigrate(&db.LfsObj{}) because the LfsObj gorm tags carry + // MySQL-specific clauses ("ON UPDATE CURRENT_TIMESTAMP") that SQLite + // rejects. The columns below cover every field the query layer reads. + require.NoError(t, gormDB.Exec(`CREATE TABLE lfs_objs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + oid TEXT, + file_name TEXT, + size INTEGER, + platform TEXT, + owner TEXT, + repo TEXT, + operator TEXT, + exist INTEGER, + update_time DATETIME, + create_time DATETIME + )`).Error) + + prev := db.Db + db.Db = gormDB + return func() { + db.Db = prev + _ = sqlDB.Close() + } +} + +// TestGetLfsFiles_PlatformFilter is a regression guard for the round-1 bugfix +// that made platform="" skip the platform filter (so the frontend "all" filter +// returns every file) while platform!="" narrows to that platform. Pinning both +// branches prevents a future refactor from silently re-breaking the "all" view. +func TestGetLfsFiles_PlatformFilter(t *testing.T) { + teardown := newSQLiteTestDB(t) + defer teardown() + + const owner, repo = "mindspore-ai", "repo" + rows := []db.LfsObj{ + {Oid: "a", Owner: owner, Repo: repo, Platform: "github", Exist: 1, Size: 10}, + {Oid: "b", Owner: owner, Repo: repo, Platform: "gitee", Exist: 1, Size: 20}, + {Oid: "c", Owner: owner, Repo: repo, Platform: "github", Exist: 0, Size: 30}, // deleted, excluded + {Oid: "d", Owner: "other-org", Repo: repo, Platform: "github", Exist: 1, Size: 40}, + } + for i := range rows { + require.NoError(t, db.Db.Create(&rows[i]).Error) + } + + s := &server{} + + // platform == "" -> no platform WHERE clause: both exist=1 platforms returned + all, err := s.getLfsFiles(owner, repo, "", 1, 10) + require.NoError(t, err) + oids := oidsOf(all) + assert.ElementsMatch(t, []string{"a", "b"}, oids) + + // platform == "github" -> platform filter applied: only the github row + gh, err := s.getLfsFiles(owner, repo, "github", 1, 10) + require.NoError(t, err) + assert.ElementsMatch(t, []string{"a"}, oidsOf(gh)) +} + +// TestCountLfsFiles_PlatformFilter mirrors the above for the count path, which +// shares the same if/else platform-filter branch and previously reported 0%. +func TestCountLfsFiles_PlatformFilter(t *testing.T) { + teardown := newSQLiteTestDB(t) + defer teardown() + + const owner, repo = "mindspore-ai", "repo" + rows := []db.LfsObj{ + {Oid: "a", Owner: owner, Repo: repo, Platform: "github", Exist: 1, Size: 10}, + {Oid: "b", Owner: owner, Repo: repo, Platform: "gitee", Exist: 1, Size: 20}, + {Oid: "c", Owner: owner, Repo: repo, Platform: "github", Exist: 0, Size: 30}, + } + for i := range rows { + require.NoError(t, db.Db.Create(&rows[i]).Error) + } + + s := &server{} + + totalAll, err := s.countLfsFiles(owner, repo, "") + require.NoError(t, err) + assert.Equal(t, int64(2), totalAll, `platform="" must not add a platform filter`) + + totalGh, err := s.countLfsFiles(owner, repo, "github") + require.NoError(t, err) + assert.Equal(t, int64(1), totalGh, `platform="github" must narrow to github rows`) +} + +func oidsOf(files []db.LfsObj) []string { + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, f.Oid) + } + return out +} diff --git a/server/platform_test.go b/server/platform_test.go new file mode 100644 index 0000000..88720fa --- /dev/null +++ b/server/platform_test.go @@ -0,0 +1,85 @@ +package server + +import ( + "errors" + "testing" + + "github.com/metalogical/BigFiles/auth" + "github.com/stretchr/testify/assert" +) + +// TestTokenForPlatform covers all three branches of tokenForPlatform: the +// github/gitcode token dispatch and the default (gitee) fallback. These tokens +// back the OID->filename backfill task (TASK5) and must map each platform to +// the right server-side credential. +func TestTokenForPlatform(t *testing.T) { + origGithub := githubDefaultToken + origAtom := atomGiteDefaultToken + origGitee := giteeDefaultToken + defer func() { + githubDefaultToken = origGithub + atomGiteDefaultToken = origAtom + giteeDefaultToken = origGitee + }() + githubDefaultToken = "gh-bot-token" + atomGiteDefaultToken = "gitcode-bot-token" + giteeDefaultToken = "gitee-bot-token" + + assert.Equal(t, "gh-bot-token", tokenForPlatform("github")) + assert.Equal(t, "gitcode-bot-token", tokenForPlatform("gitcode")) + // default branch: any unknown platform falls back to the gitee token + assert.Equal(t, "gitee-bot-token", tokenForPlatform("gitee")) + assert.Equal(t, "gitee-bot-token", tokenForPlatform("unknown-platform")) +} + +// callAuthorized invokes the authorizer returned by authorizedFn and reports +// which sentinel it produced, so we can assert which branch was taken without +// comparing function values (Go forbids comparing funcs except to nil). +func callAuthorized(s *server, owner string) string { + fn := s.authorizedFn(auth.UserInRepo{Owner: owner}) + err := fn(auth.UserInRepo{Owner: owner}) + if err == nil { + return "" + } + return err.Error() +} + +// TestAuthorizedFn_FallbackNilMap pins the fallback path: when no +// platform-specific authorizer map is configured, the generic IsAuthorized must +// be used. This was the previously-uncovered branch (50% -> now covered). +func TestAuthorizedFn_FallbackNilMap(t *testing.T) { + s := &server{ + isAuthorized: func(auth.UserInRepo) error { return errors.New("fallback") }, + isAuthorizedByPlatform: nil, + } + assert.Equal(t, "fallback", callAuthorized(s, "openeuler")) +} + +// TestAuthorizedFn_FallbackUnknownPlatform covers the case where the map is set +// but the resolved platform has no entry, so the generic authorizer is used. +func TestAuthorizedFn_FallbackUnknownPlatform(t *testing.T) { + resolved := auth.ResolvePlatform("openeuler") + s := &server{ + isAuthorized: func(auth.UserInRepo) error { return errors.New("fallback") }, + isAuthorizedByPlatform: map[string]func(auth.UserInRepo) error{ + "github": func(auth.UserInRepo) error { return errors.New("platform-gh") }, + }, + } + // sanity: the resolved platform really is missing from the map + _, present := s.isAuthorizedByPlatform[resolved] + assert.False(t, present) + assert.Equal(t, "fallback", callAuthorized(s, "openeuler")) +} + +// TestAuthorizedFn_PlatformHit covers the happy path: the resolved platform has +// a registered authorizer, which must be returned instead of the generic one. +func TestAuthorizedFn_PlatformHit(t *testing.T) { + resolved := auth.ResolvePlatform("openeuler") + s := &server{ + isAuthorized: func(auth.UserInRepo) error { return errors.New("fallback") }, + isAuthorizedByPlatform: map[string]func(auth.UserInRepo) error{ + resolved: func(auth.UserInRepo) error { return errors.New("platform-hit") }, + }, + } + assert.Equal(t, "platform-hit", callAuthorized(s, "openeuler")) +} diff --git a/server/server.go b/server/server.go index acf072f..a46693d 100644 --- a/server/server.go +++ b/server/server.go @@ -47,7 +47,8 @@ type Options struct { TTL time.Duration // defaults to 1 hour Prefix string - IsAuthorized func(auth.UserInRepo) error + IsAuthorized func(auth.UserInRepo) error + IsAuthorizedByPlatform map[string]func(auth.UserInRepo) error } func (o Options) imputeFromEnv() (Options, error) { @@ -91,12 +92,13 @@ func New(o Options) (http.Handler, error) { } s := &server{ - ttl: o.TTL, - client: client, - bucket: o.Bucket, - prefix: o.Prefix, - cdnDomain: o.CdnDomain, - isAuthorized: o.IsAuthorized, + ttl: o.TTL, + client: client, + bucket: o.Bucket, + prefix: o.Prefix, + cdnDomain: o.CdnDomain, + isAuthorized: o.IsAuthorized, + isAuthorizedByPlatform: o.IsAuthorizedByPlatform, } r := chi.NewRouter() @@ -119,7 +121,8 @@ type server struct { prefix string cdnDomain string - isAuthorized func(auth.UserInRepo) error + isAuthorized func(auth.UserInRepo) error + isAuthorizedByPlatform map[string]func(auth.UserInRepo) error } func (s *server) key(oid string) string { @@ -154,7 +157,8 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { return } - if _, err = auth.CheckRepoOwner(userInRepo); req.Operation == "upload" || err != nil { + platform := auth.ResolvePlatform(userInRepo.Owner) + if _, err = auth.CheckRepoOwnerByPlatform(platform, userInRepo); req.Operation == "upload" || err != nil { err := s.dealWithAuthError(userInRepo, w, r) if err != nil { return @@ -170,10 +174,7 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { } func addMetaData(req batch.Request, w http.ResponseWriter, userInRepo auth.UserInRepo) { - platform := "gitee" - if gitCodeSwitch { - platform = "atomGit" - } + platform := auth.PlatformForMetadata(auth.ResolvePlatform(userInRepo.Owner)) // 添加元数据 if req.Operation == "upload" { for _, object := range req.Objects { @@ -238,6 +239,17 @@ func (s *server) handleRequestObject(req batch.Request) batch.Response { return resp } +// authorizedFn returns the platform-specific authorizer, falling back to the +// generic IsAuthorized when no platform mapping is configured. +func (s *server) authorizedFn(userInRepo auth.UserInRepo) func(auth.UserInRepo) error { + if s.isAuthorizedByPlatform != nil { + if fn, ok := s.isAuthorizedByPlatform[auth.ResolvePlatform(userInRepo.Owner)]; ok && fn != nil { + return fn + } + } + return s.isAuthorized +} + func (s *server) dealWithAuthError(userInRepo auth.UserInRepo, w http.ResponseWriter, r *http.Request) error { var err error if username, password, ok := r.BasicAuth(); ok { @@ -252,7 +264,7 @@ func (s *server) dealWithAuthError(userInRepo auth.UserInRepo, w http.ResponseWr })) return errors.New("invalid username or password format") } - err = s.isAuthorized(userInRepo) + err = s.authorizedFn(userInRepo)(userInRepo) } else if authToken := r.Header.Get("Authorization"); authToken != "" { err = auth.VerifySSHAuthToken(authToken, userInRepo) } else { @@ -471,7 +483,11 @@ func (s *server) List(w http.ResponseWriter, r *http.Request) { return } - resp := s.buildListResponse(files, total) + // The repo's hosting platform is derived from its owner (ResolvePlatform), + // not from the file rows, so it stays stable regardless of the platform + // filter and is correct even when no LFS files exist yet. + repoPlatform := auth.ResolvePlatform(owner) + resp := s.buildListResponse(files, total, repoPlatform) w.Header().Set(contentType, jsonHeader) w.WriteHeader(http.StatusOK) @@ -522,9 +538,12 @@ func (s *server) getLfsFiles(owner, repo, platform string, page, limit int) ([]d func (s *server) countLfsFiles(owner, repo, platform string) (int64, error) { var total int64 - if err := db.Db.Model(&db.LfsObj{}). - Where("owner = ? AND repo = ? AND platform = ? AND exist = 1", owner, repo, platform). - Count(&total).Error; err != nil { + query := db.Db.Model(&db.LfsObj{}). + Where("owner = ? AND repo = ? AND exist = 1", owner, repo) + if platform != "" { + query = query.Where("platform = ?", platform) + } + if err := query.Count(&total).Error; err != nil { return 0, err } return total, nil @@ -536,11 +555,12 @@ type FileResponse struct { Size int `json:"size"` Oid string `json:"oid"` FileName string `json:"file_name"` + Platform string `json:"platform"` CreateTime int64 `json:"create_time"` UpdateTime int64 `json:"update_time"` } -func (s *server) buildListResponse(files []db.LfsObj, total int64) interface{} { +func (s *server) buildListResponse(files []db.LfsObj, total int64, repoPlatform string) interface{} { response := make([]FileResponse, len(files)) for i, file := range files { response[i] = FileResponse{ @@ -549,17 +569,20 @@ func (s *server) buildListResponse(files []db.LfsObj, total int64) interface{} { Size: file.Size, Oid: file.Oid, FileName: file.FileName, + Platform: file.Platform, CreateTime: file.CreateTime.Unix(), UpdateTime: file.UpdateTime.Unix(), } } return struct { - Total int `json:"total"` - Files []FileResponse `json:"files"` + Total int `json:"total"` + Platform string `json:"platform"` + Files []FileResponse `json:"files"` }{ - Total: int(total), - Files: response, + Total: int(total), + Platform: repoPlatform, + Files: response, } } @@ -592,23 +615,32 @@ func checkOidFileName() { logrus.Errorf("fetch repo list failed: %v", err) return } - token := giteeDefaultToken - if gitCodeSwitch { - token = atomGiteDefaultToken - } for _, repo := range repoList { + platform := auth.ResolvePlatform(repo.Owner) userInRepo := auth.UserInRepo{ Repo: repo.Repo, Owner: repo.Owner, - Token: token} - logrus.Infof("checkOidFileName owner:%v repo:%v", repo.Owner, repo.Repo) + Token: tokenForPlatform(platform)} + logrus.Infof("checkOidFileName owner:%v repo:%v platform:%v", repo.Owner, repo.Repo, platform) checkRepoOidName(userInRepo) } } +func tokenForPlatform(platform string) string { + switch platform { + case "github": + return githubDefaultToken + case "gitcode": + return atomGiteDefaultToken + default: + return giteeDefaultToken + } +} + func checkRepoOidName(userInRepo auth.UserInRepo) (oidFileNameMap map[string]auth.FileInfo) { + platform := auth.ResolvePlatform(userInRepo.Owner) oidFileNameMap, err := auth.GetLFSMapping(userInRepo) if err != nil { logrus.Errorf("get lfs mapping failed: %v", err) @@ -616,7 +648,7 @@ func checkRepoOidName(userInRepo auth.UserInRepo) (oidFileNameMap map[string]aut checkOidFileNameMap(oidFileNameMap, userInRepo) if strings.ToLower(userInRepo.Owner) != "src-openeuler" { logrus.Infof("after check owner:%v repo:%v, check src-openeuler", userInRepo.Owner, userInRepo.Repo) - repo, err := auth.CheckRepoOwner(userInRepo) + repo, err := auth.CheckRepoOwnerByPlatform(platform, userInRepo) if err != nil { return nil } @@ -633,6 +665,7 @@ func checkOidFileNameMap(oidFileNameMap map[string]auth.FileInfo, userInRepo aut if oidFileNameMap == nil { return } + platform := auth.PlatformForMetadata(auth.ResolvePlatform(userInRepo.Owner)) for oid, fileInfo := range oidFileNameMap { lfsObjs, err := db.SelectLfsObjByOid(oid) if err != nil { @@ -649,7 +682,7 @@ func checkOidFileNameMap(oidFileNameMap map[string]auth.FileInfo, userInRepo aut Size: int(fileInfo.Size), FileName: fileInfo.Name, Exist: 2, - Platform: "gitee", + Platform: platform, Operator: "", } if err = db.InsertLFSObj(lfsObj); err != nil { @@ -659,11 +692,11 @@ func checkOidFileNameMap(oidFileNameMap map[string]auth.FileInfo, userInRepo aut } // 检查对应oid文件在对应仓库下是否存在,如果不存在则创建对应数据 - checkLfsObjsInfo(oid, lfsObjs, fileInfo, userInRepo) + checkLfsObjsInfo(oid, lfsObjs, fileInfo, userInRepo, platform) } } -func checkLfsObjsInfo(oid string, lfsObjs []db.LfsObj, fileInfo auth.FileInfo, userInRepo auth.UserInRepo) { +func checkLfsObjsInfo(oid string, lfsObjs []db.LfsObj, fileInfo auth.FileInfo, userInRepo auth.UserInRepo, platform string) { exist := false logrus.Infof("check oid:%v info", oid) for _, lfsObj := range lfsObjs { @@ -688,7 +721,7 @@ func checkLfsObjsInfo(oid string, lfsObjs []db.LfsObj, fileInfo auth.FileInfo, u Size: int(fileInfo.Size), FileName: fileInfo.Name, Exist: 2, - Platform: "gitee", + Platform: platform, Operator: "", } if err := db.InsertLFSObj(lfsObj); err != nil { diff --git a/server/server_test.go b/server/server_test.go index 795bc66..aae1544 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -960,16 +960,18 @@ func TestBuildListResponse(t *testing.T) { s := &server{} // 调用 buildListResponse - response := s.buildListResponse(files, total) + response := s.buildListResponse(files, total, "gitee") // 进行断言,确保返回的结构符合预期 result := response.(struct { - Total int `json:"total"` - Files []FileResponse `json:"files"` + Total int `json:"total"` + Platform string `json:"platform"` + Files []FileResponse `json:"files"` }) // 验证结果 assert.Equal(t, total, int64(result.Total)) + assert.Equal(t, "gitee", result.Platform) assert.Equal(t, "owner1", result.Files[0].Owner) assert.Equal(t, "repo1", result.Files[0].Repo) assert.Equal(t, 1234, result.Files[0].Size) diff --git a/server/validate.go b/server/validate.go index 82c3b52..ea86b25 100644 --- a/server/validate.go +++ b/server/validate.go @@ -18,12 +18,14 @@ var Webhook_key string var gitCodeSwitch bool var giteeDefaultToken string var atomGiteDefaultToken string +var githubDefaultToken string func Init(cfg *config.Config) error { validateConfig := cfg.ValidateConfig gitCodeSwitch = cfg.GitCodeSwitch giteeDefaultToken = cfg.DefaultToken atomGiteDefaultToken = cfg.DefaultGitCodeToken + githubDefaultToken = cfg.GithubDefaultToken var err error Webhook_key = validateConfig.WebhookKey