From 4e9f94f6c791cc99733652d3a0d654f1a8fe38bc Mon Sep 17 00:00:00 2001 From: mendezr Date: Wed, 18 Nov 2026 18:00:37 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix:=20prime=20TechDocs=20assess?= =?UTF-8?q?ments=20with=20scoped=20reviewer=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seed source-linked assessment criteria from cncf/techdocs#424 and include workflow-scoped knowledge in dispatch kicks without leaking it into unrelated quality work. Hive-Run: hivecommons/hive#19050 Hive-Plan: reviewer-rules-workflow-injection Hive-Spec: hivecommons/hive#19050#expected Signed-off-by: mendezr --- .../fixed-techdocs-reviewer-knowledge.md | 1 + src/cmd/hive/main_eval_cycle.go | 1 + src/cmd/hive/workflow_dispatch.go | 8 +- src/docs/knowledge-entries.md | 8 +- .../knowledge/techdocs-assessment-evidence.md | 33 ++++++ .../techdocs-assessment-ownership.md | 26 +++++ src/docs/workflows/techdocs-assessment.md | 48 +++++++++ src/pkg/knowledge/scope.go | 16 +-- src/pkg/knowledge/scope_test.go | 3 + src/pkg/scheduler/kick_priming.go | 23 +++- src/pkg/scheduler/workflow_knowledge_test.go | 101 ++++++++++++++++++ 11 files changed, 256 insertions(+), 12 deletions(-) create mode 100644 changelog.d/fixed-techdocs-reviewer-knowledge.md create mode 100644 src/docs/knowledge/techdocs-assessment-evidence.md create mode 100644 src/docs/knowledge/techdocs-assessment-ownership.md create mode 100644 src/pkg/scheduler/workflow_knowledge_test.go diff --git a/changelog.d/fixed-techdocs-reviewer-knowledge.md b/changelog.d/fixed-techdocs-reviewer-knowledge.md new file mode 100644 index 0000000000..ee79ab3539 --- /dev/null +++ b/changelog.d/fixed-techdocs-reviewer-knowledge.md @@ -0,0 +1 @@ +- Workflow-dispatch kicks now receive scoped knowledge, including source-linked TechDocs reviewer rules for evidence-based assessment scores and authoritative project ownership, without applying assessment-only rules to unrelated quality work ([#19050](https://github.com/hivecommons/hive/issues/19050)). diff --git a/src/cmd/hive/main_eval_cycle.go b/src/cmd/hive/main_eval_cycle.go index 0032ce4f91..6acc6e5833 100644 --- a/src/cmd/hive/main_eval_cycle.go +++ b/src/cmd/hive/main_eval_cycle.go @@ -1060,6 +1060,7 @@ func runEvalCycle( } return workflowAgentTurnFrom(st) }, logger) { + k.Message = sched.PrimeWorkflowDispatch(k.Message, k.Dispatch) workflowMessages = append(workflowMessages, k.Message) workflowKickByMessage[workflowDispatchKickKey(k.Message)] = k } diff --git a/src/cmd/hive/workflow_dispatch.go b/src/cmd/hive/workflow_dispatch.go index 9b9bba148a..d19135583e 100644 --- a/src/cmd/hive/workflow_dispatch.go +++ b/src/cmd/hive/workflow_dispatch.go @@ -166,8 +166,9 @@ func (rt *workflowDispatchRuntime) clearPending(key string) { // workflowDispatchKick is one planned hand-off and the record its delivery // writes. type workflowDispatchKick struct { - Message scheduler.KickMessage - Entry scheduler.WorkflowDispatchEntry + Message scheduler.KickMessage + Entry scheduler.WorkflowDispatchEntry + Dispatch github.WorkflowDispatch } // workflowDispatchActive reports whether the cycle dispatches at all. @@ -351,7 +352,8 @@ func buildWorkflowDispatchKicks(cfg *config.Config, rt *workflowDispatchRuntime, } rt.markPending(c.Key, now) out = append(out, workflowDispatchKick{ - Message: scheduler.KickMessage{Agent: c.Agent, Message: msg, IssueRefs: refs}, + Dispatch: c.Dispatch, + Message: scheduler.KickMessage{Agent: c.Agent, Message: msg, IssueRefs: refs}, Entry: scheduler.NextWorkflowDispatchEntry(scheduler.WorkflowDispatchEntry{ Key: c.Key, Ref: c.Dispatch.Ref(), Profile: c.Dispatch.Target.Profile, State: c.Dispatch.State, Agent: c.Agent, By: "governor", diff --git a/src/docs/knowledge-entries.md b/src/docs/knowledge-entries.md index 6ee6a5bdfe..b0676b7cf3 100644 --- a/src/docs/knowledge-entries.md +++ b/src/docs/knowledge-entries.md @@ -43,7 +43,7 @@ A PR that edits anything under `src/scripts/` must re-pin … | `tags` | Inline list. Always include `repo:/` — that is how `knowledge.agent_scopes.repos` recognises the entry's repository. | | `source` | Where the fact came from, so a reader can trace a moved `AGENTS.md` line to exactly one entry. | | `author` | Who wrote the entry — an agent name (`developer`) or a login (alias `owner`). A `draft` reaches a kick only for the agent that matches this and has opted into drafts; see [knowledge-toc.md](knowledge-toc.md#per-agent-scopes). | -| `scope` | **Where the fact applies.** `paths` are repo-relative globs (`**` supported), `labels` are issue labels, `lanes` are base agent/lane names. An empty list means "no constraint on that dimension". | +| `scope` | **Where the fact applies.** `paths` are repo-relative globs (`**` supported), `labels` are issue labels, `lanes` are base agent/lane names or unversioned workflow model names. An empty list means "no constraint on that dimension". | Rules of thumb: @@ -72,7 +72,11 @@ trigger. At kick time Hive knows the lane and the issue labels; file paths are known only to callers that have them (the mid-task advisor, tracked separately), so a `paths`-only entry is a lookup target for an agent reading this directory from its checkout until that lands. The gate is implemented by -hivecommons/hive#17045. +hivecommons/hive#17045. Workflow-dispatch kicks additionally match `lanes` against +the bound model's unversioned name (for example `cncf-techdocs-assessment`), +without replacing the base agent lane. This keeps workflow-specific reviewer +rules available to whichever agent fills the profile, without leaking them to +the same agent's unrelated work. ## How the directory reaches the primer diff --git a/src/docs/knowledge/techdocs-assessment-evidence.md b/src/docs/knowledge/techdocs-assessment-evidence.md new file mode 100644 index 0000000000..3ae8325a17 --- /dev/null +++ b/src/docs/knowledge/techdocs-assessment-evidence.md @@ -0,0 +1,33 @@ +--- +title: cncf-techdocs-assessment reviewer rules for evidence and scoring +type: gotcha +layer: project +state: approved +tags: [repo:cncf/techdocs, techdocs, assessment, reviewer-rule] +source: https://github.com/cncf/techdocs/pull/424#discussion_r4618113440 +scope: + lanes: [cncf-techdocs-assessment] +--- + +For TechDocs assessments, apply the published criteria to observed behavior on +the requested documentation domain, not a different site CLOMonitor scans. +Record the target URL and results for each readiness criterion. An `llms.txt` +file or URL-presence test alone does not prove Markdown availability, content +parity or synchronization. Do not inflate the grade or call readiness a +“standout area” without criterion-level evidence. + +Additional reviewer corrections on the same assessment: + +- Inspect existing adoption/testimonial materials before reporting their absence + ([source](https://github.com/cncf/techdocs/pull/424#discussion_r4617291386)). +- Distinguish installed localization/analytics infrastructure from functionality + actually delivered on documentation pages + ([localization](https://github.com/cncf/techdocs/pull/424#discussion_r4617357700), + [analytics](https://github.com/cncf/techdocs/pull/424#discussion_r4617591526)). +- Assess the applicable website checklist before assigning an exemplary Sandbox + grade; documentation volume alone is insufficient + ([source](https://github.com/cncf/techdocs/pull/424#discussion_r4618143283)). +- Mark usability/accessibility “not assessed” until supporting interaction checks + are recorded. Say “No scan was performed or supplied for this assessment”, + not that no scan has ever happened + ([source](https://github.com/cncf/techdocs/pull/424#discussion_r4618183780)). diff --git a/src/docs/knowledge/techdocs-assessment-ownership.md b/src/docs/knowledge/techdocs-assessment-ownership.md new file mode 100644 index 0000000000..c06f5de312 --- /dev/null +++ b/src/docs/knowledge/techdocs-assessment-ownership.md @@ -0,0 +1,26 @@ +--- +title: cncf-techdocs-assessment ownership uses the project's authoritative roster +type: gotcha +layer: project +state: approved +tags: [repo:cncf/techdocs, techdocs, assessment, reviewer-rule, ownership] +source: https://github.com/cncf/techdocs/pull/424#discussion_r4617524979 +scope: + lanes: [cncf-techdocs-assessment] +--- + +Identify the project's authoritative maintainers roster before recommending +changes to documentation ownership. Distinguish project maintainers, docs +reviewers and CNCF technical writers; do not claim somebody reviewed an +assessment without evidence. + +For Hive Commons, the authoritative roster is `hivecommons/.project`'s +`maintainers.yaml`, not docs `OWNERS` or `CODEOWNERS`. Reconcile those files with +the current official roster and intended docs-review responsibilities instead +of merely recommending a second approver. Check the current roster and GitHub +CODEOWNERS validation rather than freezing the reviewer comment's counts or +membership as permanent facts. This project's roster is not the authority for +other projects. + +Reviewer evidence: +https://github.com/hivecommons/.project/blob/5eba57f04b9228427aef2b5c7af6d1232fd4a5bd/maintainers.yaml#L15-L25 diff --git a/src/docs/workflows/techdocs-assessment.md b/src/docs/workflows/techdocs-assessment.md index add24d90a9..1f3146c15c 100644 --- a/src/docs/workflows/techdocs-assessment.md +++ b/src/docs/workflows/techdocs-assessment.md @@ -313,6 +313,54 @@ hand-off, and `workflow.dispatch.enabled: false` stops them all (owned items sta from the generic lanes either way). `prompt_template` replaces the built-in prompt; see the design doc for its `${VAR}` placeholders. +### Assessment criteria and reviewer rules + +The assessment rubric lives in the TechDocs repository's +[`docs/analysis/criteria.md`](https://github.com/cncf/techdocs/blob/main/docs/analysis/criteria.md) +and its assessment specification (linked above). The curated workflow model defines +lifecycle states, human gates and drafter/verifier profiles, not a second scoring +rubric. Hive selects an eligible agent (which may be `quality`, as described above) +and builds its instructions with `scheduler.BuildWorkflowDispatchPrompt` or the +profile's configured `prompt_template`. + +Reviewer-stated corrections belong in the shared knowledge store as small, +source-linked entries, not only in a reply or the agent's private memory. Scope +assessment rules to the **workflow model name**, independent of its version and +of which agent fills the profile: + +```yaml +title: cncf-techdocs-assessment reviewer rule for +type: gotcha +layer: project +state: approved +tags: [repo:cncf/techdocs, techdocs, assessment, reviewer-rule] +source: https://github.com/cncf/techdocs/pull/#discussion_r +scope: + lanes: [cncf-techdocs-assessment] +``` + +This is frontmatter for a [knowledge entry](../knowledge-entries.md); follow it +with the verified rule and its project-specific limits. Preserve the originating +comment URL/id when capturing a rule. Draft rules remain excluded until approved. +Do not use `lanes: [quality]` alone: that would teach unrelated quality work an +assessment-specific rule and miss a drafter assigned to a different agent. + +Workflow hand-offs run the knowledge primer before delivery. The bound model's +unversioned name is an additional lane match and search term, alongside the base +agent lane and item labels. The usual agent knowledge scope, lifecycle filter, +relevance selection and byte cap still apply. A rule must be in a registered +vault/git source and fit the primer budget to reach the next kick. See +[registering repository knowledge](../knowledge-entries.md#how-the-directory-reaches-the-primer). + +The initial reviewer rules from `cncf/techdocs#424` live in +[`techdocs-assessment-evidence.md`](../knowledge/techdocs-assessment-evidence.md) +and [`techdocs-assessment-ownership.md`](../knowledge/techdocs-assessment-ownership.md). +They cover evidence-based readiness grading on the documentation domain and the +project's authoritative maintainer roster. They supplement rather than replace +the upstream rubric. Automated review-feedback capture and reply verification +are tracked separately in hivecommons/hive#19048; this injection path also accepts +subsequent captured entries with the same scope. + ### When the agent does not start A kick that landed only means the agent was told. An agent can decline (one whose session diff --git a/src/pkg/knowledge/scope.go b/src/pkg/knowledge/scope.go index 048e501162..1219de10bc 100644 --- a/src/pkg/knowledge/scope.go +++ b/src/pkg/knowledge/scope.go @@ -21,7 +21,7 @@ type FactScope struct { Paths []string `json:"paths,omitempty"` // Labels are issue labels, compared case-insensitively. Labels []string `json:"labels,omitempty"` - // Lanes are base agent / lane names, compared case-insensitively. + // Lanes are base agent / lane or workflow model names, compared case-insensitively. Lanes []string `json:"lanes,omitempty"` } @@ -34,14 +34,17 @@ func (s *FactScope) IsZero() bool { // scheduler fills Lane and Labels from the lane name and the primed issues; // Paths is filled only by callers that know which files are being touched. type KickContext struct { - Lane string - Labels []string - Paths []string + Lane string + // Workflow is the bound workflow model name, without its version. It is + // an additional lane match, not a replacement for the base agent lane. + Workflow string + Labels []string + Paths []string } // IsZero reports whether the context carries nothing to match on. func (kc KickContext) IsZero() bool { - return kc.Lane == "" && len(kc.Labels) == 0 && len(kc.Paths) == 0 + return kc.Lane == "" && kc.Workflow == "" && len(kc.Labels) == 0 && len(kc.Paths) == 0 } // Admits reports whether a fact carrying scope s may be shown for kc. An @@ -55,7 +58,8 @@ func (s *FactScope) Admits(kc KickContext) bool { if s.IsZero() { return true } - if kc.Lane != "" && containsFold(s.Lanes, kc.Lane) { + if (kc.Lane != "" && containsFold(s.Lanes, kc.Lane)) || + (kc.Workflow != "" && containsFold(s.Lanes, kc.Workflow)) { return true } for _, l := range kc.Labels { diff --git a/src/pkg/knowledge/scope_test.go b/src/pkg/knowledge/scope_test.go index 87f7d1050c..7e8cf71d51 100644 --- a/src/pkg/knowledge/scope_test.go +++ b/src/pkg/knowledge/scope_test.go @@ -54,6 +54,9 @@ func TestFactScopeAdmits(t *testing.T) { {"scoped fact, empty context", scoped, KickContext{}, false}, {"lane match, case-insensitive", scoped, KickContext{Lane: "Reviewer"}, true}, {"lane mismatch", scoped, KickContext{Lane: "developer"}, false}, + {"workflow matches lane scope", scoped, KickContext{Lane: "quality", Workflow: "Reviewer"}, true}, + {"workflow does not replace base lane", scoped, KickContext{Lane: "reviewer", Workflow: "techdocs"}, true}, + {"workflow mismatch", scoped, KickContext{Lane: "quality", Workflow: "techdocs"}, false}, {"label match among others", scoped, KickContext{Labels: []string{"kind/bug", "AREA/CI"}}, true}, {"label mismatch", scoped, KickContext{Labels: []string{"kind/bug"}}, false}, {"path match", scoped, KickContext{Paths: []string{"src/scripts/x.sh"}}, true}, diff --git a/src/pkg/scheduler/kick_priming.go b/src/pkg/scheduler/kick_priming.go index 29439287b0..b19bf26e2a 100644 --- a/src/pkg/scheduler/kick_priming.go +++ b/src/pkg/scheduler/kick_priming.go @@ -332,6 +332,25 @@ func (s *Scheduler) primeKnowledge(agentName string, issues []github.Issue) stri // primeKnowledgeStats is primeKnowledge without the summary log line. func (s *Scheduler) primeKnowledgeStats(agentName string, issues []github.Issue) knowledgeInjection { + return s.primeKnowledgeStatsForWorkflow(agentName, issues, "") +} + +// PrimeWorkflowDispatch adds the same scoped, lifecycle-filtered and byte-capped +// knowledge as a generic kick, plus the bound model as an additional lane. The +// model name is a search term so reviewer rules do not depend on an intake +// issue's title mentioning the assessment rubric. +func (s *Scheduler) PrimeWorkflowDispatch(msg KickMessage, d github.WorkflowDispatch) KickMessage { + model, _, _ := strings.Cut(d.Model, "@") + issues := []github.Issue{{Repo: d.Repo, Number: d.Number, Title: d.Title + " " + model, Labels: d.Labels}} + inj := s.primeKnowledgeStatsForWorkflow(msg.Agent, issues, model) + s.logKnowledgeInjection(msg.Agent, inj, 0, 0) + if inj.Section != "" { + msg.Message = inj.Section + "\n\n" + msg.Message + } + return msg +} + +func (s *Scheduler) primeKnowledgeStatsForWorkflow(agentName string, issues []github.Issue, workflow string) knowledgeInjection { s.mu.RLock() primer := s.primer cfg := s.cfg @@ -363,7 +382,9 @@ func (s *Scheduler) primeKnowledgeStats(agentName string, issues []github.Issue) } scope = &agentScope } - primed := primer.PrimeForKick(context.Background(), scope, kickContextFor(cfg, agentName, issues[:limit]), keywords) + kc := kickContextFor(cfg, agentName, issues[:limit]) + kc.Workflow = workflow + primed := primer.PrimeForKick(context.Background(), scope, kc, keywords) result, omitted := primed.Render() s.primedSlugs.Store(agentName, primed.RenderedSlugs()) inj := knowledgeInjection{ diff --git a/src/pkg/scheduler/workflow_knowledge_test.go b/src/pkg/scheduler/workflow_knowledge_test.go new file mode 100644 index 0000000000..58f7378271 --- /dev/null +++ b/src/pkg/scheduler/workflow_knowledge_test.go @@ -0,0 +1,101 @@ +package scheduler + +import ( + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/knowledge" +) + +func TestPrimeWorkflowDispatchReviewerRules(t *testing.T) { + dir := t.TempDir() + // Exercise the shipped criteria, not a second copy of their frontmatter. + for _, name := range []string{"techdocs-assessment-evidence.md", "techdocs-assessment-ownership.md"} { + b, err := os.ReadFile(filepath.Join("..", "..", "docs", "knowledge", name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), b, 0o644); err != nil { + t.Fatal(err) + } + } + write := func(name, state, lane, body string) { + t.Helper() + text := "---\ntitle: cncf-techdocs-assessment " + name + "\ntype: gotcha\nlayer: project\nstate: " + state + "\nsource: https://github.com/cncf/techdocs/pull/424#discussion_r4617524979\nscope:\n lanes: [" + lane + "]\n---\n" + body + if err := os.WriteFile(filepath.Join(dir, name+".md"), []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + // A subsequently captured rule uses the same scope as the seeded criteria. + write("captured", "approved", "cncf-techdocs-assessment", "Captured reviewer rule sentinel") + write("draft", "draft", "cncf-techdocs-assessment", "Unapproved reviewer rule sentinel") + write("elsewhere", "approved", "other-workflow", "Other workflow rule sentinel") + write("base", "approved", "quality", "Base quality rule sentinel") + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + store, err := knowledge.NewFileStore(dir, "reviewer-rules", logger) + if err != nil { + t.Fatal(err) + } + newSched := func(maxBytes int) *Scheduler { + s := newScheduler() + p := knowledge.NewPrimer(nil, knowledge.PrimerConfig{MaxFacts: 25, MaxBytes: maxBytes}, logger) + p.AddFileStore("reviewer-rules", store, knowledge.LayerProject) + s.SetPrimer(p) + return s + } + d := github.WorkflowDispatch{Repo: "cncf/techdocs", Number: 395, Title: "Review Acme", Model: "cncf-techdocs-assessment@curated-2026.11.2"} + msg := KickMessage{Agent: "quality", Message: "Original workflow instructions", IssueRefs: []string{d.Ref()}} + s := newSched(16384) + got := s.PrimeWorkflowDispatch(msg, d) + for _, want := range []string{"# Relevant Knowledge", "requested documentation domain", "authoritative roster", "Captured reviewer rule sentinel", "Base quality rule sentinel"} { + if !strings.Contains(got.Message, want) { + t.Errorf("workflow kick missing %q: %s", want, got.Message) + } + } + for _, unwanted := range []string{"Unapproved reviewer rule sentinel", "Other workflow rule sentinel"} { + if strings.Contains(got.Message, unwanted) { + t.Errorf("workflow kick leaked %q", unwanted) + } + } + if !strings.HasSuffix(got.Message, msg.Message) || got.Agent != msg.Agent || strings.Join(got.IssueRefs, ",") != d.Ref() { + t.Fatalf("priming changed dispatch instructions or identity: %+v", got) + } + // Criteria follow the workflow even when a different agent fills its profile. + otherAgent := msg + otherAgent.Agent = "scanner" + if got := s.PrimeWorkflowDispatch(otherAgent, d); !strings.Contains(got.Message, "Captured reviewer rule sentinel") { + t.Fatal("changing the profile's agent lost reviewer rules") + } + // A different version of the same model uses the same criteria. + d.Model = "cncf-techdocs-assessment@future" + if got := s.PrimeWorkflowDispatch(msg, d); !strings.Contains(got.Message, "Captured reviewer rule sentinel") { + t.Fatal("model version change lost reviewer rules") + } + d.Model = "other-workflow@1" + if got := s.PrimeWorkflowDispatch(msg, d); strings.Contains(got.Message, "Captured reviewer rule sentinel") { + t.Fatal("TechDocs rule leaked to another workflow on the same quality agent") + } + // Generic quality work must not inherit assessment-only criteria either. + if got := s.primeKnowledge("quality", []github.Issue{{Title: "cncf-techdocs-assessment"}}); strings.Contains(got, "Captured reviewer rule sentinel") { + t.Fatal("TechDocs rule leaked to a generic quality kick") + } + d.Model = "cncf-techdocs-assessment@curated-2026.11.2" + // Agent-level repository filters and the rendering cap still apply. + s.cfg.Knowledge.AgentScopes = map[string]config.KnowledgeAgentScope{"quality": {Repos: []string{"unrelated/repo"}}} + if got := s.PrimeWorkflowDispatch(msg, d); strings.Contains(got.Message, "authoritative roster") { + t.Fatal("workflow priming bypassed agent repository scope") + } + capped := newSched(512).PrimeWorkflowDispatch(msg, d) + if len(capped.Message)-len(msg.Message) > 514 { + t.Fatal("workflow knowledge exceeded its byte cap") + } + if got := newScheduler().PrimeWorkflowDispatch(msg, d); got.Message != msg.Message { + t.Fatal("disabled primer changed the workflow kick") + } +}