Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-18516-remote-items-bot-followup.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Reference repos: the own-item follow-up now also routes comments and reviews on issues and PRs the hive opened as its GitHub App bot (`project.remote_write_identity`), listing each repo under the login its writes appear as and reading it with that repo's own credential (#18516).
17 changes: 12 additions & 5 deletions src/cmd/hive/reference_repos_identity_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,15 +208,22 @@ func TestReferenceRepoIdentity_PerRepoCredential(t *testing.T) {
}
}
}
// The own-item follow-up reads under the owner token, so it watches only
// the owner-identity repos.
// The own-item follow-up watches every write-tier repo under the login
// its writes appear as: the bot's items on the bot repo, the owner's on
// the owner repo.
res, err := buildReferenceRepoPolicies(context.Background(), refs, withIdentity(testReferenceRepoDeps(), config.RemoteWriteIdentityBot, app), testLogger())
if err != nil {
t.Fatal(err)
}
repos, login := remoteItemRepos(res.repos)
if len(repos) != 1 || repos[0] != "acme/site" || login != testOwnerLogin {
t.Fatalf("follow-up must watch only owner-identity repos, got %v as %q", repos, login)
targets := remoteItemTargets(res.repos)
want := map[string]string{"hivecommons/docs": testBotLogin, "acme/site": testOwnerLogin}
if len(targets) != len(want) {
t.Fatalf("follow-up targets = %+v", targets)
}
for _, tg := range targets {
if want[tg.Repo] != tg.Login {
t.Fatalf("follow-up for %s must list items by %q, got %q", tg.Repo, want[tg.Repo], tg.Login)
}
}
}

Expand Down
63 changes: 37 additions & 26 deletions src/cmd/hive/remote_items.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,17 @@ import (
//
// Every reference-repo pass (boot, config reload, the derive poller, an
// owner-identity change) ends by polling the write-tier reference repos for
// the issues and PRs the hive opened there as the owner and routing new
// feedback to the agents that opened them. The poller is bounded by design:
// it lists creator=<owner> only and reads comments on tracked items only.
// the issues and PRs the hive opened there (as the owner, or as the App bot
// on a repo that writes as the bot, #18516) and routing new feedback to the
// agents that opened them. The poller is bounded by design: it lists
// creator=<that login> only and reads comments on tracked items only.
// Nothing here scans a remote backlog, kicks from a remote queue or runs a
// stage on a remote item.

// newRemoteItemsPoller builds the poller over the owner token source.
// newRemoteItemsPoller builds the poller over the reference-repo credentials:
// each repo's proxy policy token when a proxy is wired, else the owner token.
func (b *boot) newRemoteItemsPoller(rt *referenceRepoRuntime) *remoteitems.Poller {
if rt == nil || rt.deps.ownerToken == nil {
if rt == nil || (rt.deps.ownerToken == nil && b.githubProxy == nil) {
return nil
}
apiURL := ""
Expand All @@ -31,9 +33,24 @@ func (b *boot) newRemoteItemsPoller(rt *referenceRepoRuntime) *remoteitems.Polle
apiURL = auth.APIURL()
}
}
fetch := &github.RemoteItemsFetcher{Token: rt.deps.ownerToken, APIURL: apiURL}
if b.githubProxy != nil {
// Each watched repo is read under its own reference-repo credential,
// the one the proxy injects on its writes: the scoped bot token on a
// repo that writes as the App bot, the owner's token (or the
// write_token_file) elsewhere (hivecommons/hive#18516).
proxy := b.githubProxy
fetch.TokenFor = func(repo string) (string, bool) {
pol, ok := proxy.ReferenceRepoPolicyFor(repo)
if !ok || pol.Token == nil {
return "", false
}
return pol.Token()
}
}
return &remoteitems.Poller{
Store: remoteitems.NewStore(""),
Fetch: &github.RemoteItemsFetcher{Token: rt.deps.ownerToken, APIURL: apiURL},
Fetch: fetch,
Marker: github.HiveMarkerAgent,
ResolveAgent: func(name string) string { return resolveMarkerAgent(b.cfg, name) },
Logger: b.logger,
Expand Down Expand Up @@ -65,43 +82,37 @@ func resolveMarkerAgent(cfg *config.Config, name string) string {
return ""
}

// remoteItemRepos returns the reference repos the poller watches: write tier
// with a resolved credential login, which is the owner's login. Entries that
// write as the App bot (project.remote_write_identity) are left out: the
// poller reads under the owner token and lists creator=<owner>, so it would
// never find the bot's items there.
func remoteItemRepos(refs []config.ReferenceRepo) (repos []string, login string) {
// remoteItemTargets returns the reference repos the poller watches: write
// tier with a resolved credential login, each under the login its writes
// appear as. That is the App bot's on a repo that writes as the bot
// (project.remote_write_identity, hivecommons/hive#18516), the owner's or the
// write_token_file user's elsewhere.
func remoteItemTargets(refs []config.ReferenceRepo) []remoteitems.Target {
var out []remoteitems.Target
for _, rr := range refs {
if rr.ReadOnly() || rr.CredentialState != config.ReferenceCredentialWriteTier || rr.CredentialLogin == "" {
continue
}
if rr.CredentialIdentity == config.ReferenceIdentityBot {
continue
}
repos = append(repos, rr.Repo)
if login == "" {
login = rr.CredentialLogin
}
out = append(out, remoteitems.Target{Repo: rr.Repo, Login: rr.CredentialLogin})
}
return repos, login
return out
}

// pollRemoteItems runs one follow-up pass over refs. Entries whose
// credential is a write_token_file are listed under that file's login too
// only when it matches the owner's; a mixed set follows the first login.
// pollRemoteItems runs one follow-up pass over refs, each repo listed under
// the login its writes appear as and read under its own credential.
func (b *boot) pollRemoteItems(ctx context.Context, rt *referenceRepoRuntime, refs []config.ReferenceRepo) {
if rt == nil || rt.remoteItems == nil || ctx == nil || ctx.Err() != nil {
return
}
repos, login := remoteItemRepos(refs)
if len(repos) == 0 || login == "" {
targets := remoteItemTargets(refs)
if len(targets) == 0 {
return
}
var resumer remoteitems.Resumer
if b.agentMgr != nil {
resumer = b.agentMgr
}
out := rt.remoteItems.Run(ctx, repos, login, resumer, time.Now())
out := rt.remoteItems.RunTargets(ctx, targets, resumer, time.Now())
if out.Added+out.Pruned+out.Routed+out.Queued+out.Unrouted+out.ListErrs+out.FetchErrs > 0 {
b.logger.Info("remote items: follow-up pass", "repos", out.Repos, "tracked", out.Tracked, "added", out.Added, "pruned", out.Pruned,
"routed", out.Routed, "queued", out.Queued, "unrouted", out.Unrouted, "list_errors", out.ListErrs, "fetch_errors", out.FetchErrs)
Expand Down
16 changes: 11 additions & 5 deletions src/cmd/hive/remote_items_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"testing"

"github.com/hivecommons/hive/pkg/config"
"github.com/hivecommons/hive/pkg/remoteitems"
)

// hivecommons/hive#18199: which repos the remote-item pass watches and how a
Expand All @@ -15,12 +16,17 @@ func TestRemoteItemRepos(t *testing.T) {
{Repo: "hivecommons/hive", Write: []string{"issues", "prs"}, CredentialState: config.ReferenceCredentialWriteTier, CredentialLogin: "nate-double-u"},
{Repo: "hivecommons/site", Write: []string{"issues"}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier, CredentialLogin: "nate-double-u"},
}
repos, login := remoteItemRepos(refs)
if len(repos) != 2 || repos[0] != "hivecommons/hive" || repos[1] != "hivecommons/site" || login != "nate-double-u" {
t.Fatalf("repos=%v login=%q", repos, login)
targets := remoteItemTargets(refs)
if len(targets) != 2 || targets[0] != (remoteitems.Target{Repo: "hivecommons/hive", Login: "nate-double-u"}) || targets[1] != (remoteitems.Target{Repo: "hivecommons/site", Login: "nate-double-u"}) {
t.Fatalf("targets=%+v", targets)
}
if repos, login := remoteItemRepos(refs[:2]); len(repos) != 0 || login != "" {
t.Fatalf("read-only and downgraded entries must not be watched: %v %q", repos, login)
if targets := remoteItemTargets(refs[:2]); len(targets) != 0 {
t.Fatalf("read-only and downgraded entries must not be watched: %+v", targets)
}
// hivecommons/hive#18516: a bot-identity repo is watched under the bot.
bot := config.ReferenceRepo{Repo: "hivecommons/docs", Write: []string{"issues", "prs"}, DerivedFrom: "cncf/techdocs#395", CredentialState: config.ReferenceCredentialWriteTier, CredentialLogin: "hivecommons-hive[bot]", CredentialIdentity: config.ReferenceIdentityBot}
if targets := remoteItemTargets([]config.ReferenceRepo{bot}); len(targets) != 1 || targets[0].Login != "hivecommons-hive[bot]" {
t.Fatalf("bot repo must be watched under the bot login: %+v", targets)
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1299,11 +1299,11 @@ How it works and what it never does:
- **Per-repo status.** Settings shows, on each write entry, which identity will actually post and why: `posts as hivecommons-hive[bot]: App installed on <org>`, `posts as <owner>: App not installed on <org>`, `read-only: connect your GitHub account`, or `read-only: bot only, App not installed on <org>`. A change that would leave a hand-written write entry without a credential is refused before it is saved, with the error boot would stop on.
- **Upgrade.** Hives that never set the key read as `bot`. Where the App is not installed, nothing changes. A granted repo whose owner installed the App now gets posts as the bot instead of the owner. Choose *Me* to keep the previous behaviour everywhere.

The own-item follow-up below reads under the owner token, so for now it follows only the items opened as the owner.
The own-item follow-up below follows the items opened under either identity, each repo read with its own credential.

**Fork-and-PR.** The owner's account cannot push to a repository it does not own, so a PR against a remote repo goes through the owner's fork: `gh repo fork <owner>/<name> --clone=false`, push the topic branch to `https://<host>/<login>/<name>`, then `gh pr create --repo <owner>/<name> --head <login>:<branch>`. The proxy allows exactly that (fork creation, pushes to the fork with its default branch protected, the PR against the remote) and refuses everything else on the fork. The kick spells the commands out, so `git push` and `gh pr create` need no agent-side special casing.

**Own-item follow-up.** Remote repos send no webhooks and are not enumerated, so the hive polls them: it lists the open issues and PRs created by the owner login, keeps the ones whose body carries a hive marker (the `— hive:` attribution trailer or the `🐝 **Hive Agent**` footer), fetches new comments, review comments and reviews, and delivers them to the agent the marker names, into its live session or as a `🌐 REMOTE ITEM FOLLOW-UP` section on its next kick. State is in `/data/turn/remote-items.json` (`HIVE_REMOTE_ITEMS_FILE` overrides). Nothing else happens on the remote: no backlog scan, no queue, no stage engine ([#18199](https://github.com/hivecommons/hive/issues/18199)).
**Own-item follow-up.** Remote repos send no webhooks and are not enumerated, so the hive polls them: it lists the open issues and PRs created by the login each repo writes as (the owner's, or the App bot's on a repo that writes as the bot, [#18516](https://github.com/hivecommons/hive/issues/18516)), reading each repo under its own reference-repo credential, keeps the ones whose body carries a hive marker (the `— hive:` attribution trailer or the `🐝 **Hive Agent**` footer), fetches new comments, review comments and reviews, and delivers them to the agent the marker names, into its live session or as a `🌐 REMOTE ITEM FOLLOW-UP` section on its next kick. State is in `/data/turn/remote-items.json` (`HIVE_REMOTE_ITEMS_FILE` overrides). Nothing else happens on the remote: no backlog scan, no queue, no stage engine ([#18199](https://github.com/hivecommons/hive/issues/18199)).

What agents see: the `📚 REFERENCE REPOS` block appends `(from owner/repo#N)` to each derived line, `(from owner/repo#N; read-only: <reason>)` when the credential fell back, and `(read-only this pass: <reason>)` while an App lookup is unresolved. The proxy rules are the ones in the table above: a derived entry unlocks opening issues and PRs on that repo and nothing else, and the read-only fallback unlocks nothing. On the dashboard, Settings shows derived entries read-only with a link to the request issue and the credential state, and the API answers an edit of one with `409`.

Expand Down
35 changes: 27 additions & 8 deletions src/pkg/github/remote_items.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,16 @@ import (
// missing token is an error rather than an unauthenticated read (a public
// listing would work, but the poller must never run on a disconnected
// identity: there is nothing to attribute to).
//
// TokenFor, when set, picks the token per repository instead
// (hivecommons/hive#18516): a repo that writes as the App bot is read under
// that repo's own minted bot token, so a hive whose owner never connected an
// account still follows up on what its bot opened. When set it is the only
// source; Token serves callers that do not set it.
type RemoteItemsFetcher struct {
Token OwnerTokenSource
APIURL string
Token OwnerTokenSource
TokenFor func(repo string) (string, bool)
APIURL string
}

// remoteItemsPageSize is GitHub's largest page.
Expand All @@ -32,13 +39,25 @@ const remoteItemsMaxPages = 3

var _ remoteitems.Fetcher = (*RemoteItemsFetcher)(nil)

func (f *RemoteItemsFetcher) client() (*gh.Client, error) {
if f == nil || f.Token == nil {
return nil, fmt.Errorf("remote items: no owner token source")
func (f *RemoteItemsFetcher) client(repo string) (*gh.Client, error) {
if f == nil {
return nil, fmt.Errorf("remote items: no token source")
}
if f.TokenFor != nil {
// Authoritative when set: a repo that writes as the bot is never
// read under the owner's token instead (bot_only never uses it).
tok, ok := f.TokenFor(repo)
if !ok || tok == "" {
return nil, fmt.Errorf("remote items: no token for %s", repo)
}
return newTokenClient(tok, f.APIURL), nil
}
if f.Token == nil {
return nil, fmt.Errorf("remote items: no token for %s", repo)
}
tok, ok := f.Token()
if !ok {
return nil, fmt.Errorf("remote items: owner identity not connected")
return nil, fmt.Errorf("remote items: no usable token for %s", repo)
}
return newTokenClient(tok, f.APIURL), nil
}
Expand All @@ -55,7 +74,7 @@ func splitSlug(repo string) (owner, name string, err error) {
// GitHub's issues listing returns both; a PR is told apart by its
// pull_request field.
func (f *RemoteItemsFetcher) ListOwnOpenItems(ctx context.Context, repo, login string) ([]remoteitems.RemoteItem, error) {
client, err := f.client()
client, err := f.client(repo)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -105,7 +124,7 @@ func (f *RemoteItemsFetcher) ListOwnOpenItems(ctx context.Context, repo, login s
// CommentsSince lists the issue comments on repo#number created after since
// and, for a PR, its review comments and reviews too, oldest first.
func (f *RemoteItemsFetcher) CommentsSince(ctx context.Context, repo string, number int, isPR bool, since time.Time) ([]remoteitems.Comment, error) {
client, err := f.client()
client, err := f.client(repo)
if err != nil {
return nil, err
}
Expand Down
69 changes: 69 additions & 0 deletions src/pkg/github/remote_items_tokenfor_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package github

import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
)

// hivecommons/hive#18516: the remote-items fetcher reads each repo under the
// token TokenFor gives for it (the scoped bot token on a bot-identity repo),
// and when TokenFor is set it never falls back to the owner's token.
func TestRemoteItemsFetcher_TokenForPerRepo(t *testing.T) {
var (
mu sync.Mutex
seen = map[string]string{}
)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
seen[r.URL.Path] = r.Header.Get("Authorization")
mu.Unlock()
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[]`))
}))
t.Cleanup(srv.Close)

ownerReads := 0
f := &RemoteItemsFetcher{
Token: func() (string, bool) { ownerReads++; return "gho_owner", true },
TokenFor: func(repo string) (string, bool) {
switch repo {
case "hivecommons/docs":
return "ghs_bot_docs", true
case "acme/site":
return "gho_owner", true
}
return "", false
},
APIURL: srv.URL,
}
ctx := context.Background()
for _, repo := range []string{"hivecommons/docs", "acme/site"} {
if _, err := f.ListOwnOpenItems(ctx, repo, "someone"); err != nil {
t.Fatalf("%s: %v", repo, err)
}
}
if got := seen["/repos/hivecommons/docs/issues"]; !strings.HasSuffix(got, "ghs_bot_docs") {
t.Fatalf("bot repo read with %q, want the bot token", got)
}
if got := seen["/repos/acme/site/issues"]; !strings.HasSuffix(got, "gho_owner") {
t.Fatalf("owner repo read with %q, want the owner token", got)
}
// No token for the repo: an error, never the owner's token instead.
if _, err := f.ListOwnOpenItems(ctx, "other/lib", "someone"); err == nil {
t.Fatal("a repo without a token must error")
}
if _, err := f.CommentsSince(ctx, "other/lib", 1, true, time.Time{}); err == nil {
t.Fatal("comments on a repo without a token must error")
}
if ownerReads != 0 {
t.Fatalf("TokenFor is authoritative: the owner token source was read %d times", ownerReads)
}
if _, ok := seen["/repos/other/lib/issues"]; ok {
t.Fatal("no request may leave without a token")
}
}
42 changes: 38 additions & 4 deletions src/pkg/remoteitems/remoteitems.go
Original file line number Diff line number Diff line change
Expand Up @@ -267,12 +267,44 @@ type Poller struct {
Logger *slog.Logger
}

// Run performs one pass over repos for the owner login: list, track, prune,
// fetch feedback, deliver. Errors on one repo or item never stop the pass.
// resumer may be nil (everything is queued as a handoff).
// Target is one watched repo and the login the hive's writes there appear
// under: the hive owner's, or the App bot's when the repo writes as the bot
// (project.remote_write_identity, hivecommons/hive#18516).
type Target struct {
Repo string
Login string
}

// Run performs one pass over repos for one login; see RunTargets.
func (p *Poller) Run(ctx context.Context, repos []string, login string, resumer Resumer, now time.Time) Outcome {
targets := make([]Target, 0, len(repos))
for _, r := range repos {
targets = append(targets, Target{Repo: r, Login: login})
}
return p.RunTargets(ctx, targets, resumer, now)
}

// RunTargets performs one pass over targets, each listed under its own
// login: list, track, prune, fetch feedback, deliver. Errors on one repo or
// item never stop the pass. resumer may be nil (everything is queued as a
// handoff). A target without a login is skipped; a repo not in targets
// drops its tracked items.
func (p *Poller) RunTargets(ctx context.Context, targets []Target, resumer Resumer, now time.Time) Outcome {
var out Outcome
if p == nil || p.Store == nil || p.Fetch == nil || p.Marker == nil || strings.TrimSpace(login) == "" || len(repos) == 0 {
loginFor := map[string]string{}
var repos []string
for _, t := range targets {
repo, login := strings.TrimSpace(t.Repo), strings.TrimSpace(t.Login)
if repo == "" || login == "" {
continue
}
if _, dup := loginFor[strings.ToLower(repo)]; dup {
continue
}
loginFor[strings.ToLower(repo)] = login
repos = append(repos, repo)
}
if p == nil || p.Store == nil || p.Fetch == nil || p.Marker == nil || len(repos) == 0 {
return out
}
logger := p.Logger
Expand All @@ -294,6 +326,7 @@ func (p *Poller) Run(ctx context.Context, repos []string, login string, resumer
continue
}
out.Repos++
login := loginFor[strings.ToLower(repo)]
items, err := p.Fetch.ListOwnOpenItems(ctx, repo, login)
if err != nil {
out.ListErrs++
Expand Down Expand Up @@ -355,6 +388,7 @@ func (p *Poller) Run(ctx context.Context, repos []string, login string, resumer
continue
}
var fresh []Comment
login := loginFor[strings.ToLower(it.Repo)]
for _, c := range comments {
if !c.CreatedAt.After(it.Cursor) || strings.EqualFold(c.Author, login) || strings.HasSuffix(c.Author, "[bot]") || strings.TrimSpace(c.Body) == "" {
continue
Expand Down
Loading
Loading