Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-modelfamily-family-provider-prefix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Model family: a model id with a provider or gateway path (`anthropic/claude-opus-5-5`, `openai-codex/gpt-6.1-terra:high`) now reads as its vendor family (`claude`, `gpt`) instead of `anthropic/claude` or `openai`. The evidence summary's by-family counts and the reviewer-accuracy card's author families stop filing those authors under `other`, and `review_models.exclude_author_family` and the same/cross-family split now treat a prefixed and a bare spelling of the same vendor as the same family, so a "cross-family" review can no longer be same-vendor. Verdicts already stamped `same_family` keep their stored value.
3 changes: 2 additions & 1 deletion src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,8 @@ agents:

`exclude_author_model` defaults to true. `exclude_author_family` compares the
normalized vendor prefix before the first dash (`claude`, `gpt`, `gemini`,
`grok`, ...). If no pool entry is eligible, `fallback` is `pinned` (use the
`grok`, ...), after dropping a provider path and `:variant` suffix, so
`anthropic/claude-opus-5-5` and `claude-sonnet-5` are the same family. If no pool entry is eligible, `fallback` is `pinned` (use the
agent's configured `model`), `skip` (omit the PR from that kick), or
`requires_human` (keep the PR visible with a human-review marker). Reviewers
record the delegated sub-agent's model as `review_model` in the verdict artifact,
Expand Down
9 changes: 6 additions & 3 deletions src/pkg/modelfamily/modelfamily.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,13 @@ func allDigits(s string) bool {

func isDigit(c byte) bool { return c >= '0' && c <= '9' }

// Family is the normalized vendor prefix before the first dash: claude, gpt,
// gemini, grok, ... Unknown stays Unknown.
// Family is the vendor prefix before the first dash of the Canonical id:
// claude, gpt, gemini, grok, ... A provider or gateway path and a ":variant"
// suffix are dropped first, so "anthropic/claude-opus-5-5" and
// "openai-codex/gpt-6.1-sol:high" read as claude and gpt, like their bare
// spellings. Unknown stays Unknown.
func Family(model string) string {
m := Normalize(model)
m := Canonical(model)
if m == Unknown {
return Unknown
}
Expand Down
47 changes: 47 additions & 0 deletions src/pkg/modelfamily/modelfamily_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,53 @@ func TestCanonicalFoldsSpellings(t *testing.T) {
}
}

// A provider or gateway path and a ":variant" suffix name how the model was
// reached, not who made it, so they must not change the family or bucket.
func TestFamilyIgnoresProviderPrefixAndVariant(t *testing.T) {
cases := []struct{ in, family, bucket string }{
{"claude-opus-5-5", "claude", BucketClaude},
{"anthropic/claude-opus-5-5", "claude", BucketClaude},
{"aws/claude-haiku-4-5", "claude", BucketClaude},
{"anthropic/claude-sonnet-5:max", "claude", BucketClaude},
{"gpt-6.1-sol", "gpt", BucketGPT},
{"openai-codex/gpt-6.1-sol", "gpt", BucketGPT},
{"openai-codex/gpt-6.1-terra:high", "gpt", BucketGPT},
{"github-copilot/gpt-5.6-terra", "gpt", BucketGPT},
{"gemini-3.8-flash", "gemini", BucketGemini},
{"github-copilot/gemini-3.8-flash", "gemini", BucketGemini},
{"google/gemini-3.8-flash-low", "gemini", BucketGemini},
{"qwen-3", "qwen", BucketOther},
{"lemonade/qwen-3", "qwen", BucketOther},
{"rits/zai-org/glm-5-3", "glm", BucketOther},
{"anthropic/", Unknown, Unknown},
{"-", Unknown, Unknown},
}
for _, c := range cases {
if got := Family(c.in); got != c.family {
t.Errorf("Family(%q)=%q want %q", c.in, got, c.family)
}
if got := Bucket(c.in); got != c.bucket {
t.Errorf("Bucket(%q)=%q want %q", c.in, got, c.bucket)
}
}
}

func TestSameFamilyAcrossProviderSpellings(t *testing.T) {
pairs := []struct{ a, b string }{
{"anthropic/claude-opus-5-5", "claude-sonnet-5"},
{"openai-codex/gpt-6.1-terra:high", "gpt-5.6-sol"},
{"github-copilot/gemini-3.8-flash", "gemini-4-argon"},
}
for _, p := range pairs {
if same, known := SameFamily(p.a, p.b); !same || !known {
t.Errorf("SameFamily(%q, %q): got same=%v known=%v, want same vendor", p.a, p.b, same, known)
}
}
if same, known := SameFamily("anthropic/claude-opus-5-5", "openai-codex/gpt-6.1-sol"); same || !known {
t.Errorf("prefixed cross family: got same=%v known=%v", same, known)
}
}

func TestSameFamily(t *testing.T) {
if same, known := SameFamily("claude-fable-5", "claude-opus-4-6"); !same || !known {
t.Fatalf("same family: got same=%v known=%v", same, known)
Expand Down
Loading