Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions service/submitqueue/orchestrator/server/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ go_library(
"//submitqueue/entity:go_default_library",
"//submitqueue/extension/baserevision:go_default_library",
"//submitqueue/extension/baserevision/git:go_default_library",
"//submitqueue/extension/baserevision/github:go_default_library",
"//submitqueue/extension/buildrunner:go_default_library",
"//submitqueue/extension/buildrunner/buildkite:go_default_library",
"//submitqueue/extension/buildrunner/fake:go_default_library",
Expand Down
10 changes: 10 additions & 0 deletions service/submitqueue/orchestrator/server/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,16 @@ type tangoAnalyzerConfig struct {
// Strategy is how Tango computes the base target graph: "unset", "shell"
// or "native". Required.
Strategy string `yaml:"strategy"`
// GitHub names the branch whose tip is the base revision. Required with
// the github change provider, which keeps no clone to read it from.
GitHub *tangoGitHubConfig `yaml:"github"`
}

// tangoGitHubConfig locates a queue's target branch on GitHub.
type tangoGitHubConfig struct {
Owner string `yaml:"owner"`
Repo string `yaml:"repo"`
Branch string `yaml:"branch"`
}

// scorerConfig selects how a queue ranks candidate speculation paths. The
Expand Down
46 changes: 44 additions & 2 deletions service/submitqueue/orchestrator/server/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,7 @@ func TestLoadProfilesConfig_QueueScorerBaseReplacesDefaultBase(t *testing.T) {
func TestNewTangoAnalyzerFactory(t *testing.T) {
key := tangoAnalyzerKey{
Analyzer: analyzerConfig{Type: analyzerTypeTango, Tango: &tangoAnalyzerConfig{Remote: "origin", Strategy: "unset"}},
Git: gitProviderConfig{RepoPath: "/var/repos/x", Remote: "origin", Target: "main"},
Git: &gitProviderConfig{RepoPath: "/var/repos/x", Remote: "origin", Target: "main"},
}
connection, err := tangoclient.Dial("127.0.0.1:0")
require.NoError(t, err)
Expand Down Expand Up @@ -968,7 +968,49 @@ func TestNewTangoAnalyzerFactory(t *testing.T) {
}
}

func TestNewProfiles_TangoRequiresTheGitChangeProvider(t *testing.T) {
func TestNewTangoAnalyzerFactory_GitHub(t *testing.T) {
connection, err := tangoclient.Dial("127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = connection.Close() })
db, _, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })

tests := []struct {
name string
branch *tangoGitHubConfig
token string
wantErr bool
}{
{name: "with a branch and a token", branch: &tangoGitHubConfig{Owner: "org", Repo: "repo", Branch: "main"}, token: "secret"},
{name: "without the branch block", token: "secret", wantErr: true},
{name: "with an incomplete branch block", branch: &tangoGitHubConfig{Owner: "org", Repo: "repo"}, token: "secret", wantErr: true},
{name: "without a token", branch: &tangoGitHubConfig{Owner: "org", Repo: "repo", Branch: "main"}, wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("TEST_TANGO_GITHUB_TOKEN", tt.token)
key := tangoAnalyzerKey{
Analyzer: analyzerConfig{Type: analyzerTypeTango, Tango: &tangoAnalyzerConfig{Remote: "origin", Strategy: "unset", GitHub: tt.branch}},
GitHub: &githubProviderConfig{TokenEnv: "TEST_TANGO_GITHUB_TOKEN", BaseURL: defaultGitHubBaseURL},
}
b := &profileBuilder{scope: tally.NoopScope, tango: tangoDeps{DB: db, Connection: connection, Cache: tango.NewSignatureCache(1 << 20)}}

factory, err := b.newTangoAnalyzerFactory(key, "defaults")

if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
analyzer, err := factory.For(conflict.Config{QueueName: "q"})
require.NoError(t, err)
assert.NotNil(t, analyzer)
})
}
}

func TestNewProfiles_TangoRequiresTheGitOrGitHubChangeProvider(t *testing.T) {
cfg, err := loadProfilesConfig(writeProfiles(t, `
tango: {address: "127.0.0.1:0", signatureCacheBytes: 1048576}
defaults:
Expand Down
85 changes: 63 additions & 22 deletions service/submitqueue/orchestrator/server/profiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
"github.com/uber/submitqueue/submitqueue/entity"
"github.com/uber/submitqueue/submitqueue/extension/baserevision"
baserevisiongit "github.com/uber/submitqueue/submitqueue/extension/baserevision/git"
baserevisiongithub "github.com/uber/submitqueue/submitqueue/extension/baserevision/github"
"github.com/uber/submitqueue/submitqueue/extension/buildrunner"
buildkiterunner "github.com/uber/submitqueue/submitqueue/extension/buildrunner/buildkite"
buildfake "github.com/uber/submitqueue/submitqueue/extension/buildrunner/fake"
Expand Down Expand Up @@ -310,10 +311,14 @@ func (b *profileBuilder) build(cfg queueProfileConfig, where string) (Profile, e
}
var analyzer conflict.Factory
if cfg.Analyzer.Type == analyzerTypeTango {
if cfg.ChangeProvider.Git == nil {
return Profile{}, fmt.Errorf("%s: the tango analyzer needs the git change provider, got %q", where, cfg.ChangeProvider.Type)
key := tangoAnalyzerKey{Analyzer: cfg.Analyzer, Git: cfg.ChangeProvider.Git}
if cfg.ChangeProvider.Type == changeProviderTypeGitHub {
key.GitHub = cfg.ChangeProvider.GitHub
}
analyzer, err = reuse(b, tangoAnalyzerKey{Analyzer: cfg.Analyzer, Git: *cfg.ChangeProvider.Git}, where, b.newTangoAnalyzerFactory)
if key.Git == nil && key.GitHub == nil {
return Profile{}, fmt.Errorf("%s: the tango analyzer needs the git or github change provider, got %q", where, cfg.ChangeProvider.Type)
}
analyzer, err = reuse(b, key, where, b.newTangoAnalyzerFactory)
} else {
analyzer, err = reuse(b, cfg.Analyzer, where, b.newAnalyzerFactory)
}
Expand Down Expand Up @@ -739,16 +744,18 @@ func (b *profileBuilder) newAnalyzerFactory(cfg analyzerConfig, where string) (c
}), nil
}

// tangoAnalyzerKey keys the tango analyzer on both the analyzer block and the
// git block of its queue, because the base revision comes from that clone.
// tangoAnalyzerKey keys the tango analyzer on the analyzer block and on the
// change provider block its base revision comes from. Exactly one of Git and
// GitHub is set.
type tangoAnalyzerKey struct {
Analyzer analyzerConfig
Git gitProviderConfig
Git *gitProviderConfig
GitHub *githubProviderConfig
}

// newTangoAnalyzerFactory builds a factory for tango analyzers that read the
// base revision from the queue's own clone. Validation guarantees the
// process-level pieces are set.
// base revision from the queue's own clone or from the GitHub API. Validation
// guarantees the process-level pieces are set.
func (b *profileBuilder) newTangoAnalyzerFactory(key tangoAnalyzerKey, where string) (conflict.Factory, error) {
if b.tango.Connection == nil || b.tango.Cache == nil || b.tango.DB == nil {
return nil, fmt.Errorf("%s: the tango analyzer needs a tango connection, signature cache and database", where)
Expand All @@ -757,28 +764,62 @@ func (b *profileBuilder) newTangoAnalyzerFactory(key tangoAnalyzerKey, where str
if err != nil {
return nil, fmt.Errorf("%s: %w", where, err)
}

repo, ok := b.repos[key.Git.RepoPath]
if !ok {
return nil, fmt.Errorf("%s: no change repository provisioned at %q", where, key.Git.RepoPath)
baseRevisions, err := b.newBaseRevisions(key, where)
if err != nil {
return nil, err
}
workspaces := cloneWorkspaces{}
target := baserevisiongit.Target{Repo: key.Git.RepoPath, Remote: key.Git.Remote, Branch: key.Git.Target}

return analyzerFunc(func(c conflict.Config) (conflict.Analyzer, error) {
return tango.New(tango.Params{
Signatures: tangomysql.NewSignatureStore(b.tango.DB, b.scope.SubScope("tango"), c.QueueName),
Cache: b.tango.Cache,
Client: client,
BaseRevisions: lockedBaseRevisions{
Resolver: baserevisiongit.New(workspaces, map[string]baserevisiongit.Target{c.QueueName: target}),
lock: repo,
},
Changes: b.resolver,
Signatures: tangomysql.NewSignatureStore(b.tango.DB, b.scope.SubScope("tango"), c.QueueName),
Cache: b.tango.Cache,
Client: client,
BaseRevisions: baseRevisions(c.QueueName),
Changes: b.resolver,
}), nil
}), nil
}

// newBaseRevisions returns a function building the base revision resolver of a
// queue, reading from the queue's clone when the git change provider is used and
// from the GitHub API otherwise. The clone and the HTTP client are set up once
// here.
func (b *profileBuilder) newBaseRevisions(key tangoAnalyzerKey, where string) (func(queue string) baserevision.Resolver, error) {
if key.Git != nil {
repo, ok := b.repos[key.Git.RepoPath]
if !ok {
return nil, fmt.Errorf("%s: no change repository provisioned at %q", where, key.Git.RepoPath)
}
target := baserevisiongit.Target{Repo: key.Git.RepoPath, Remote: key.Git.Remote, Branch: key.Git.Target}
return func(queue string) baserevision.Resolver {
return lockedBaseRevisions{
Resolver: baserevisiongit.New(cloneWorkspaces{}, map[string]baserevisiongit.Target{queue: target}),
lock: repo,
}
}, nil
}

branch := key.Analyzer.Tango.GitHub
if branch == nil || branch.Owner == "" || branch.Repo == "" || branch.Branch == "" {
return nil, fmt.Errorf("%s: the tango analyzer on the github change provider needs tango.github with an owner, repo and branch", where)
}
token, ok := tokenFrom(key.GitHub.TokenEnv)
if !ok {
return nil, fmt.Errorf("%s: the tango analyzer on the github change provider needs %s to be set", where, key.GitHub.TokenEnv)
}
client, err := b.githubClient(key.GitHub.BaseURL, key.GitHub.Timeout, token, where)
if err != nil {
return nil, err
}
target := baserevisiongithub.Target{Owner: branch.Owner, Repo: branch.Repo, Branch: branch.Branch}
return func(queue string) baserevision.Resolver {
return baserevisiongithub.New(baserevisiongithub.Params{
HTTPClient: client,
Targets: map[string]baserevisiongithub.Target{queue: target},
})
}, nil
}

// lockedBaseRevisions holds the clone's lock around each lookup, because the
// change provider works in the same clone.
type lockedBaseRevisions struct {
Expand Down
18 changes: 18 additions & 0 deletions submitqueue/extension/baserevision/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Base Revision Extension

Vendor-agnostic source of the commit a queue's new batches are analyzed against.

## Interfaces

### Resolver

Returns the full commit SHA of the tip of a queue's target branch, as of the call. A queue without a target is an error, and so is any failure to read the tip. A caller must not fall back to an earlier value, because the revision names the code a batch is measured against. Implementations must be safe for concurrent use.

## Implementations

- `git` fetches the target branch into a clone through any `gitworkspace.Factory`, so where git runs is the deployment's choice. Calls that share a repository are serialized, because concurrent fetches into one clone can fail on a ref lock.
- `github` reads the branch from the GitHub REST API, so it needs no clone. It takes an HTTP client that already carries the API root and credentials, as the GitHub change provider does.

A deployer can supply its own implementation, for example one backed by an internal source control service.

The Tango conflict analyzer is the only consumer today. The orchestrator wiring picks the implementation from the queue's change provider.
14 changes: 13 additions & 1 deletion submitqueue/extension/baserevision/baserevision.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,19 @@ package baserevision

//go:generate mockgen -source=baserevision.go -destination=mock/baserevision_mock.go -package=mock

import "context"
import (
"context"
"regexp"
)

// fullCommitSHA matches the SHA-1 and SHA-256 object formats.
var fullCommitSHA = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`)

// IsFullCommitSHA reports whether s is a complete commit SHA. A base revision
// must be one, because the consumers of it reject an abbreviated SHA.
func IsFullCommitSHA(s string) bool {
return fullCommitSHA.MatchString(s)
}

// Resolver returns the commit a queue's new batches are analyzed against.
type Resolver interface {
Expand Down
7 changes: 1 addition & 6 deletions submitqueue/extension/baserevision/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ package git
import (
"context"
"fmt"
"regexp"
"strings"
"sync"

Expand All @@ -30,10 +29,6 @@ import (

const defaultRemote = "origin"

// fullCommitSHA matches the SHA-1 and SHA-256 object formats. Tango needs the
// full SHA, so an abbreviated one is rejected.
var fullCommitSHA = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`)

// Target locates the branch whose tip is a queue's base revision.
type Target struct {
// Repo identifies the repository to the workspace factory.
Expand Down Expand Up @@ -103,7 +98,7 @@ func (r resolver) Resolve(ctx context.Context, queue string) (string, error) {
}

sha := strings.TrimSpace(outputs[1].Stdout)
if !fullCommitSHA.MatchString(sha) {
if !baserevision.IsFullCommitSHA(sha) {
return "", fmt.Errorf("git returned %q, not a full commit SHA", sha)
}
return sha, nil
Expand Down
23 changes: 23 additions & 0 deletions submitqueue/extension/baserevision/github/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
load("@rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["github.go"],
importpath = "github.com/uber/submitqueue/submitqueue/extension/baserevision/github",
visibility = ["//visibility:public"],
deps = [
"//platform/http:go_default_library",
"//submitqueue/extension/baserevision:go_default_library",
],
)

go_test(
name = "go_default_test",
srcs = ["github_test.go"],
embed = [":go_default_library"],
deps = [
"//platform/http:go_default_library",
"@com_github_stretchr_testify//assert:go_default_library",
"@com_github_stretchr_testify//require:go_default_library",
],
)
Loading
Loading