Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -636,6 +636,7 @@ GO_GENERATE_PACKAGES := \
./stovepipe/extension/sourcecontrol/... \
./stovepipe/extension/storage/... \
./submitqueue/core/changeset/... \
./submitqueue/extension/baserevision/... \
./submitqueue/extension/buildrunner/... \
./submitqueue/extension/changeprovider/... \
./submitqueue/extension/conflict/... \
Expand Down
3 changes: 2 additions & 1 deletion service/submitqueue/orchestrator/server/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ go_library(
"//service/submitqueue/core:go_default_library",
"//submitqueue/core/changeset:go_default_library",
"//submitqueue/entity:go_default_library",
"//submitqueue/extension/baserevision:go_default_library",
"//submitqueue/extension/baserevision/git:go_default_library",
"//submitqueue/extension/buildrunner:go_default_library",
"//submitqueue/extension/buildrunner/buildkite:go_default_library",
"//submitqueue/extension/buildrunner/fake:go_default_library",
Expand All @@ -59,7 +61,6 @@ go_library(
"//submitqueue/extension/conflict/none:go_default_library",
"//submitqueue/extension/conflict/pathoverlap:go_default_library",
"//submitqueue/extension/conflict/tango:go_default_library",
"//submitqueue/extension/conflict/tango/baserevision:go_default_library",
"//submitqueue/extension/conflict/tango/mysql:go_default_library",
"//submitqueue/extension/conflict/tango/tangoclient:go_default_library",
"//submitqueue/extension/queuepause:go_default_library",
Expand Down
27 changes: 20 additions & 7 deletions service/submitqueue/orchestrator/server/profiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"database/sql"
"fmt"
nethttp "net/http"
"sync"

orchstorage "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage"

Expand All @@ -36,6 +37,8 @@ import (
"github.com/uber/submitqueue/platform/http"
"github.com/uber/submitqueue/submitqueue/core/changeset"
"github.com/uber/submitqueue/submitqueue/entity"
"github.com/uber/submitqueue/submitqueue/extension/baserevision"
baserevisiongit "github.com/uber/submitqueue/submitqueue/extension/baserevision/git"
"github.com/uber/submitqueue/submitqueue/extension/buildrunner"
buildkiterunner "github.com/uber/submitqueue/submitqueue/extension/buildrunner/buildkite"
buildfake "github.com/uber/submitqueue/submitqueue/extension/buildrunner/fake"
Expand All @@ -52,7 +55,6 @@ import (
"github.com/uber/submitqueue/submitqueue/extension/conflict/none"
"github.com/uber/submitqueue/submitqueue/extension/conflict/pathoverlap"
"github.com/uber/submitqueue/submitqueue/extension/conflict/tango"
"github.com/uber/submitqueue/submitqueue/extension/conflict/tango/baserevision"
tangomysql "github.com/uber/submitqueue/submitqueue/extension/conflict/tango/mysql"
"github.com/uber/submitqueue/submitqueue/extension/conflict/tango/tangoclient"
"github.com/uber/submitqueue/submitqueue/extension/speculation/allocator/sticky"
Expand Down Expand Up @@ -761,24 +763,35 @@ func (b *profileBuilder) newTangoAnalyzerFactory(key tangoAnalyzerKey, where str
return nil, fmt.Errorf("%s: no change repository provisioned at %q", where, key.Git.RepoPath)
}
workspaces := cloneWorkspaces{}
target := baserevision.Target{Repo: key.Git.RepoPath, Remote: key.Git.Remote, Branch: key.Git.Target}
target := baserevisiongit.Target{Repo: key.Git.RepoPath, Remote: key.Git.Remote, Branch: key.Git.Target}

return analyzerFunc(func(c conflict.Config) (conflict.Analyzer, error) {
fromGit := baserevision.FromGit(workspaces, map[string]baserevision.Target{c.QueueName: target})
return tango.New(tango.Params{
Signatures: tangomysql.NewSignatureStore(b.tango.DB, b.scope.SubScope("tango"), c.QueueName),
Cache: b.tango.Cache,
Client: client,
BaseRevision: func(ctx context.Context, queue string) (string, error) {
repo.Lock()
defer repo.Unlock()
return fromGit(ctx, queue)
BaseRevisions: lockedBaseRevisions{
Resolver: baserevisiongit.New(workspaces, map[string]baserevisiongit.Target{c.QueueName: target}),
lock: repo,
},
Changes: b.resolver,
}), nil
}), nil
}

// lockedBaseRevisions holds the clone's lock around each lookup, because the
// change provider works in the same clone.
type lockedBaseRevisions struct {
baserevision.Resolver
lock sync.Locker
}

func (l lockedBaseRevisions) Resolve(ctx context.Context, queue string) (string, error) {
l.lock.Lock()
defer l.lock.Unlock()
return l.Resolver.Resolve(ctx, queue)
}

// cloneWorkspaces serves each repo, named by its clone directory, as a local
// git workspace.
type cloneWorkspaces struct{}
Expand Down
8 changes: 8 additions & 0 deletions submitqueue/extension/baserevision/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
load("@rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["baserevision.go"],
importpath = "github.com/uber/submitqueue/submitqueue/extension/baserevision",
visibility = ["//visibility:public"],
)
28 changes: 28 additions & 0 deletions submitqueue/extension/baserevision/baserevision.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package baserevision

//go:generate mockgen -source=baserevision.go -destination=mock/baserevision_mock.go -package=mock

import "context"

// Resolver returns the commit a queue's new batches are analyzed against.
type Resolver interface {
// Resolve returns the full commit SHA of the tip of the queue's target
// branch as of the call. An error means the tip could not be determined,
// and a caller must not fall back to an earlier value. Implementations
// must be safe for concurrent use.
Resolve(ctx context.Context, queue string) (string, error)
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,18 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["baserevision.go"],
importpath = "github.com/uber/submitqueue/submitqueue/extension/conflict/tango/baserevision",
srcs = ["git.go"],
importpath = "github.com/uber/submitqueue/submitqueue/extension/baserevision/git",
visibility = ["//visibility:public"],
deps = ["//platform/extension/gitworkspace:go_default_library"],
deps = [
"//platform/extension/gitworkspace:go_default_library",
"//submitqueue/extension/baserevision:go_default_library",
],
)

go_test(
name = "go_default_test",
srcs = ["baserevision_test.go"],
srcs = ["git_test.go"],
embed = [":go_default_library"],
deps = [
"//platform/extension/gitworkspace:go_default_library",
Expand Down
110 changes: 110 additions & 0 deletions submitqueue/extension/baserevision/git/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package git resolves the base revision of a queue by asking a git workspace
// for the tip of the queue's target branch. Any gitworkspace.Factory works, so
// where git runs is the deployment's choice.
package git

import (
"context"
"fmt"
"regexp"
"strings"
"sync"

"github.com/uber/submitqueue/platform/extension/gitworkspace"
"github.com/uber/submitqueue/submitqueue/extension/baserevision"
)

const defaultRemote = "origin"

// fullCommitSHA matches the SHA-1 and SHA-256 object formats. Tango needs the
// full SHA, so an abbreviated one is rejected.
var fullCommitSHA = regexp.MustCompile(`^([0-9a-f]{40}|[0-9a-f]{64})$`)

// Target locates the branch whose tip is a queue's base revision.
type Target struct {
// Repo identifies the repository to the workspace factory.
Repo string

// Remote is the git remote the branch is fetched from. It defaults to
// origin.
Remote string

// Branch is the target branch.
Branch string
}

type resolver struct {
factory gitworkspace.Factory
targets map[string]Target
locks map[string]*sync.Mutex
}

// New returns a Resolver that fetches the target branch of the queue and
// returns its tip as a full commit SHA. A queue without a target, or any git
// failure, is an error. Calls that share a Repo are serialized, because
// concurrent fetches into one clone can fail on a ref lock and overwrite each
// other's FETCH_HEAD.
func New(factory gitworkspace.Factory, targets map[string]Target) baserevision.Resolver {
locks := make(map[string]*sync.Mutex, len(targets))
for _, target := range targets {
locks[target.Repo] = &sync.Mutex{}
}
return resolver{factory: factory, targets: targets, locks: locks}
}

func (r resolver) Resolve(ctx context.Context, queue string) (string, error) {
target, ok := r.targets[queue]
if !ok {
return "", fmt.Errorf("no target branch configured for queue %s", queue)
}
remote := target.Remote
if remote == "" {
remote = defaultRemote
}

lock := r.locks[target.Repo]
lock.Lock()
defer lock.Unlock()

workspace, err := r.factory.For(ctx, target.Repo)
if err != nil {
return "", fmt.Errorf("failed to open a workspace for repo %s: %w", target.Repo, err)
}
defer workspace.Close()

outputs, err := workspace.Exec([]gitworkspace.Command{
{Alias: "fetch", Bin: "git", Args: []string{"fetch", remote, target.Branch}},
{Alias: "tip", Bin: "git", Args: []string{"rev-parse", "FETCH_HEAD^{commit}"}},
})
if err != nil {
return "", fmt.Errorf("failed to run git in repo %s: %w", target.Repo, err)
}
for _, out := range outputs {
if out.ExitCode != 0 {
return "", fmt.Errorf("git %s failed with exit code %d: %s", out.Alias, out.ExitCode, strings.TrimSpace(out.Stderr))
}
}
if len(outputs) != 2 {
return "", fmt.Errorf("expected 2 git outputs, got %d", len(outputs))
}

sha := strings.TrimSpace(outputs[1].Stdout)
if !fullCommitSHA.MatchString(sha) {
return "", fmt.Errorf("git returned %q, not a full commit SHA", sha)
}
return sha, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package baserevision
package git

import (
"context"
Expand All @@ -38,7 +38,7 @@ const (

var errTestFailure = errors.New("dependency down")

func TestFromGit(t *testing.T) {
func TestResolver_Resolve(t *testing.T) {
tests := []struct {
name string
queue string
Expand Down Expand Up @@ -142,7 +142,7 @@ func TestFromGit(t *testing.T) {
}
}

got, err := FromGit(factory, map[string]Target{testQueue: tt.target})(context.Background(), tt.queue)
got, err := New(factory, map[string]Target{testQueue: tt.target}).Resolve(context.Background(), tt.queue)

if tt.wantErr {
require.Error(t, err)
Expand All @@ -154,7 +154,7 @@ func TestFromGit(t *testing.T) {
}
}

func TestFromGit_RepoLocks(t *testing.T) {
func TestResolver_Resolve_RepoLocks(t *testing.T) {
validOutputs := []gitworkspace.Output{{Alias: "fetch"}, {Alias: "tip", Stdout: testSHA}}

t.Run("calls on one repo never overlap", func(t *testing.T) {
Expand All @@ -172,7 +172,7 @@ func TestFromGit_RepoLocks(t *testing.T) {
inFlight.Add(-1)
return validOutputs, nil
}).AnyTimes()
baseRevision := FromGit(factory, map[string]Target{
resolver := New(factory, map[string]Target{
"main-queue": {Repo: testRepo, Branch: "main"},
"release-queue": {Repo: testRepo, Branch: "release"},
})
Expand All @@ -186,7 +186,7 @@ func TestFromGit_RepoLocks(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
_, err := baseRevision(context.Background(), queue)
_, err := resolver.Resolve(context.Background(), queue)
assert.NoError(t, err)
}()
}
Expand All @@ -209,7 +209,7 @@ func TestFromGit_RepoLocks(t *testing.T) {
})
workspace.EXPECT().Close().Return(nil)
}
baseRevision := FromGit(factory, map[string]Target{
resolver := New(factory, map[string]Target{
"queue-x": {Repo: "repo-x", Branch: "main"},
"queue-y": {Repo: "repo-y", Branch: "main"},
})
Expand All @@ -219,7 +219,7 @@ func TestFromGit_RepoLocks(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
_, err := baseRevision(context.Background(), queue)
_, err := resolver.Resolve(context.Background(), queue)
assert.NoError(t, err)
}()
}
Expand Down
9 changes: 9 additions & 0 deletions submitqueue/extension/baserevision/mock/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
load("@rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["baserevision_mock.go"],
importpath = "github.com/uber/submitqueue/submitqueue/extension/baserevision/mock",
visibility = ["//visibility:public"],
deps = ["@org_uber_go_mock//gomock:go_default_library"],
)
Loading
Loading