diff --git a/changes/unreleased/Changed-20260913-184045.yaml b/changes/unreleased/Changed-20260913-184045.yaml new file mode 100644 index 0000000..85f60b1 --- /dev/null +++ b/changes/unreleased/Changed-20260913-184045.yaml @@ -0,0 +1,5 @@ +kind: Changed +body: The published fidelity tiers are stated against two denominators — the services DevCloud promises depth on, and every service it registers — so the long tail it routes to stop SDK calls escaping to a billable account no longer reads as a drop in hand-written coverage +time: 2026-09-13T18:40:45.000000+09:00 +custom: + Issue: "166" diff --git a/cmd/devcloud/coverage_test.go b/cmd/devcloud/coverage_test.go index 3b92007..d3c1fa1 100644 --- a/cmd/devcloud/coverage_test.go +++ b/cmd/devcloud/coverage_test.go @@ -57,20 +57,38 @@ func coverageRow(t *testing.T, doc, label string) int { // tierRowPattern matches one row of the per-operation table in docs/coverage.md: // -// | `hand-verified` | 4,496 | +// | `hand-verified` | 4,497 | 4,528 | // -// Thousands separators are inside capture group 1: the doc is written for a +// Two capture groups, because the table publishes two denominators: the serving +// target first, then the whole registered fleet. Reading only the first column +// would silently repoint this gate at the narrower claim. +// +// Thousands separators are inside the capture groups: the doc is written for a // reader, and the gate reads what the reader sees rather than asking the doc to // be machine-shaped. tierRow strips them, and the updater writes them back. func tierRowPattern(label string) *regexp.Regexp { - return regexp.MustCompile("(?m)^\\|\\s*`" + regexp.QuoteMeta(label) + "`\\s*\\|\\s*([\\d,]+)\\s*\\|") + return regexp.MustCompile("(?m)^\\|\\s*`" + regexp.QuoteMeta(label) + + "`\\s*\\|\\s*([\\d,]+)\\s*\\|\\s*([\\d,]+)\\s*\\|") } -// totalKnownPattern matches the summed row of that same table. -var totalKnownPattern = regexp.MustCompile(`(?m)^\|\s*\*\*total known\*\*\s*\|\s*\*\*([\d,]+)\*\*\s*\|`) +// totalKnownPattern matches the summed row of that same table, in both columns. +var totalKnownPattern = regexp.MustCompile( + `(?m)^\|\s*\*\*total known\*\*\s*\|\s*\*\*([\d,]+)\*\*\s*\|\s*\*\*([\d,]+)\*\*\s*\|`) -// tierRow returns the operation count the manifest table publishes for a tier. -func tierRow(t *testing.T, doc, label string) int { +// handVerifiedSharePattern matches the share row of that same table: +// +// | **hand-verified share** | **36.2%** | **23.6%** | +// +// The label carries no backticks, so tierRowPattern("hand-verified") cannot +// reach it — the two rows name the same tier and only their shape tells them +// apart. Capture groups hold the number without its percent sign, so the +// updater rewrites 36.2 and leaves the % where it is. +var handVerifiedSharePattern = regexp.MustCompile( + `(?m)^\|\s*\*\*hand-verified share\*\*\s*\|\s*\*\*([\d.]+)%\*\*\s*\|\s*\*\*([\d.]+)%\*\*\s*\|`) + +// tierRow returns the operation counts the manifest table publishes for a tier: +// the serving target first, then all registered services. +func tierRow(t *testing.T, doc, label string) (target, all int) { t.Helper() matches := tierRowPattern(label).FindAllStringSubmatch(doc, -1) @@ -78,11 +96,28 @@ func tierRow(t *testing.T, doc, label string) int { t.Fatalf("docs/coverage.md: found %d rows for tier %q, want exactly 1", len(matches), label) } - n, err := strconv.Atoi(strings.ReplaceAll(matches[0][1], ",", "")) - if err != nil { - t.Fatalf("docs/coverage.md: tier %q has an unreadable number %q", label, matches[0][1]) + parse := func(column int) int { + n, err := strconv.Atoi(strings.ReplaceAll(matches[0][column], ",", "")) + if err != nil { + t.Fatalf("docs/coverage.md: tier %q column %d has an unreadable number %q", + label, column, matches[0][column]) + } + return n } - return n + return parse(1), parse(2) +} + +// shareTenths is the hand-verified share of a surface, in tenths of a percent. +// +// Integer tenths rather than a float because the gate and the updater compare +// the rendered string: a float tolerance would let the page and the binary +// disagree in the digit the page actually shows. Rounded half up, which is what +// a reader assumes "36.2%" means. +func shareTenths(hand, total int) int { + if total == 0 { + return 0 + } + return (2000*hand + total) / (2 * total) } // servedCounts splits the manifest the way docs/coverage.md publishes it. @@ -117,6 +152,15 @@ type figures struct { compatTested int // registered minus the pinned boto3 exclusions tiers map[fidelity.Tier]int totalKnown int + + // The same two counts over the serving target alone. docs/coverage.md + // publishes both denominators because reading one as the other is the + // mistake the page exists to prevent: the long tail was registered so a + // call cannot leave for a billable account, not because DevCloud promises + // to serve it well, and folding its operations into the total turns + // "we routed 226 more services" into what reads as a fidelity regression. + targetTiers map[fidelity.Tier]int + targetTotalKnown int } // derivedFigures reads every derivable published number out of the binary. @@ -133,11 +177,18 @@ func derivedFigures(t *testing.T) figures { serving: serving, registeredOnly: registeredOnly, tiers: map[fidelity.Tier]int{}, + targetTiers: map[fidelity.Tier]int{}, } - for _, svc := range fidelity.Services { + target := servingTargetServices(t) + for id, svc := range fidelity.Services { + inTarget := target[id] for _, tier := range svc.Operations { f.tiers[tier]++ f.totalKnown++ + if inTarget { + f.targetTiers[tier]++ + f.targetTotalKnown++ + } } } f.compatTested = f.registered - len(loadCompatExclusions(t)) @@ -261,9 +312,14 @@ func TestPublishedOperationTiersMatchTheManifest(t *testing.T) { fidelity.TierAutoCRUD, fidelity.TierUnimplemented, } { - if got, want := tierRow(t, doc, string(tier)), counts[tier]; got != want { - t.Errorf("docs/coverage.md publishes %d %s operations, the manifest holds %d", - got, tier, want) + gotTarget, gotAll := tierRow(t, doc, string(tier)) + if want := f.targetTiers[tier]; gotTarget != want { + t.Errorf("docs/coverage.md publishes %d %s operations inside the serving target, "+ + "the manifest holds %d", gotTarget, tier, want) + } + if want := counts[tier]; gotAll != want { + t.Errorf("docs/coverage.md publishes %d %s operations across all registered "+ + "services, the manifest holds %d", gotAll, tier, want) } } @@ -271,9 +327,183 @@ func TestPublishedOperationTiersMatchTheManifest(t *testing.T) { if published == nil { t.Fatal("docs/coverage.md: the 'total known' row did not parse") } - want, _ := strconv.Atoi(strings.ReplaceAll(published[1], ",", "")) - if want != total { - t.Errorf("docs/coverage.md publishes %d known operations, the manifest holds %d", want, total) + gotTarget, _ := strconv.Atoi(strings.ReplaceAll(published[1], ",", "")) + gotAll, _ := strconv.Atoi(strings.ReplaceAll(published[2], ",", "")) + if gotTarget != f.targetTotalKnown { + t.Errorf("docs/coverage.md publishes %d known operations inside the serving target, "+ + "the manifest holds %d", gotTarget, f.targetTotalKnown) + } + if gotAll != total { + t.Errorf("docs/coverage.md publishes %d known operations, the manifest holds %d", gotAll, total) + } +} + +// TestPublishedFidelityShareMatchesTheManifest gates the number the PRD called a +// regression and the measurement called a denominator. +// +// The hand-verified share over all 431 registered services is 23.6%, and over +// the 205-service serving target it is 36.2% — the same share it was before the +// long tail was registered. Publishing only the first invites reading a routing +// decision as a depth regression, and "offsetting" it costs roughly 2,450 +// hand-written operations that nobody asked for. So both are published, and both +// are gated: a page that states one without the other, or states either wrongly, +// fails here. +// +// Asserted as rendered strings rather than as floats. The page shows one decimal +// place, so that is the precision the claim is made at, and comparing anything +// finer would fail on a digit no reader can see. +func TestPublishedFidelityShareMatchesTheManifest(t *testing.T) { + raw, err := os.ReadFile(coveragePath) + if err != nil { + t.Fatalf("read the published coverage claim: %v", err) + } + + matches := handVerifiedSharePattern.FindAllStringSubmatch(string(raw), -1) + if len(matches) != 1 { + t.Fatalf("docs/coverage.md: found %d hand-verified share rows, want exactly 1. "+ + "The two-denominator table is what this gate reads; if it was restructured, "+ + "move the pattern deliberately rather than letting the share stop being checked.", + len(matches)) + } + + f := derivedFigures(t) + for _, c := range []struct { + column int + surface string + hand int + total int + }{ + {1, "the serving target", f.targetTiers[fidelity.TierHandVerified], f.targetTotalKnown}, + {2, "all registered services", f.tiers[fidelity.TierHandVerified], f.totalKnown}, + } { + want := formatTenths(shareTenths(c.hand, c.total)) + if got := matches[0][c.column]; got != want { + t.Errorf("docs/coverage.md publishes a hand-verified share of %s%% over %s; "+ + "%d of %d operations is %s%%", got, c.surface, c.hand, c.total, want) + } + } +} + +// longTailPattern and longTailHandVerifiedPattern match the two figures the +// paragraph under the tier table states and the table itself does not: how many +// operations the long tail brought with it, and how many of those are +// hand-written. +// +// Both are the difference between the table's two columns, so both move the week +// an operation is promoted anywhere outside the serving target. proseRequired +// exists for figures a tool cannot write; these two it can, so they are gated +// and rewritten like the cells above them rather than left as prose that is +// correct on the day it is typed. +// +// Anchored on a distinctive clause rather than pinned to the whole sentence: the +// wording stays free to change, and a reword that drops the anchor matches +// nothing — which fails here, rather than disabling the check in silence. +var ( + longTailPattern = regexp.MustCompile(`with them ([\d,]+) operations`) + longTailHandVerifiedPattern = regexp.MustCompile("([\\d,]+) are `hand-verified`") +) + +// TestPublishedLongTailProseMatchesTheManifest gates the figures that live in +// the paragraph under the tier table rather than in it. +// +// The table's own cells have been gated since the two denominators were split. +// The paragraph explaining them was not, and it states the same arithmetic in +// words: "6,794 operations, of those 31 are hand-verified". A promotion inside +// the long tail moves both, the table follows the binary, and the sentence +// underneath keeps the old numbers while every gate stays green — which is the +// drift TestOtherDocsQuoteTheSameFigure was written for, one paragraph lower. +func TestPublishedLongTailProseMatchesTheManifest(t *testing.T) { + raw, err := os.ReadFile(coveragePath) + if err != nil { + t.Fatalf("read the published coverage claim: %v", err) + } + doc := string(raw) + + f := derivedFigures(t) + for _, c := range []struct { + what string + pattern *regexp.Regexp + want int + }{ + {"operations outside the serving target", longTailPattern, + f.totalKnown - f.targetTotalKnown}, + {"of those that are hand-verified", longTailHandVerifiedPattern, + f.tiers[fidelity.TierHandVerified] - f.targetTiers[fidelity.TierHandVerified]}, + } { + matches := c.pattern.FindAllStringSubmatch(doc, -1) + if len(matches) != 1 { + t.Fatalf("docs/coverage.md: found %d statements of %q, want exactly 1. "+ + "The paragraph under the tier table is what this gate reads; if it was "+ + "reworded, move the pattern deliberately rather than letting the figure "+ + "stop being checked.", len(matches), c.what) + } + + got, err := strconv.Atoi(strings.ReplaceAll(matches[0][1], ",", "")) + if err != nil { + t.Fatalf("docs/coverage.md: %q has an unreadable number %q", c.what, matches[0][1]) + } + if got != c.want { + t.Errorf("docs/coverage.md states %d %s; the two columns differ by %d", + got, c.what, c.want) + } + } +} + +// manifestPath is the second page that states the fidelity share. See +// docs/fidelity-manifest.md. +const manifestPath = "../../docs/fidelity-manifest.md" + +// manifestSharePattern matches the share where docs/fidelity-manifest.md states +// it in a sentence: +// +// `hand-verified` is 36.2% of the operations inside the [serving target](…) +// and 23.6% of every operation DevCloud knows about +// +// The gap between the two figures spans a line break and a Markdown link, so it +// admits `.` where quotedFigurePattern does not — "coverage.md#the-target" is +// inside it. It still excludes digits and is still bounded, which is what stops +// a loose reader from becoming a loose writer: no other number can be captured, +// and a reword long enough to break the bound fails rather than matching wrongly. +var manifestSharePattern = regexp.MustCompile( + "`hand-verified` is ([\\d.]+)% of the[^\\d]{0,80}?and ([\\d.]+)% of") + +// TestFidelityManifestQuotesTheSameShare is TestOtherDocsQuoteTheSameFigure for +// the figure this PRD added. +// +// The share is gated on docs/coverage.md and restated here, and a restatement no +// gate reads is the exact shape of the drift that put "148 registered / 117 +// serving" on both front pages for three milestones. One promotion in the long +// tail moves the share, the weekly sync rewrites coverage.md, and this page goes +// on publishing the old percentage unless something compares them. +func TestFidelityManifestQuotesTheSameShare(t *testing.T) { + raw, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read the fidelity manifest page: %v", err) + } + + matches := manifestSharePattern.FindAllStringSubmatch(string(raw), -1) + if len(matches) != 1 { + t.Fatalf("docs/fidelity-manifest.md: found %d statements of the hand-verified "+ + "share, want exactly 1. The page restates a gated figure; restore the "+ + "phrasing or move the pattern deliberately.", len(matches)) + } + + f := derivedFigures(t) + for _, c := range []struct { + column int + surface string + hand int + total int + }{ + {1, "the serving target", f.targetTiers[fidelity.TierHandVerified], f.targetTotalKnown}, + {2, "all registered services", f.tiers[fidelity.TierHandVerified], f.totalKnown}, + } { + want := formatTenths(shareTenths(c.hand, c.total)) + if got := matches[0][c.column]; got != want { + t.Errorf("docs/fidelity-manifest.md states a hand-verified share of %s%% over %s; "+ + "docs/coverage.md publishes %s%%, and %d of %d operations is %s%%", + got, c.surface, want, c.hand, c.total, want) + } } } @@ -424,6 +654,21 @@ func TestPublishedTargetTableMatchesTheBinary(t *testing.T) { "%d registered minus a serving target of %d is %d. One of the three "+ "numbers moved without the others.", got, registered, serving, want) } + + // The three numbers above are internally consistent whatever they say, which + // is the half of the table this gate checked. The other half is that the + // target is the same set the tier table divides by: the "Serving target" + // column is counted over servingTargetServices, and nothing tied its size to + // the number published here. AWS shipping a service the demand study never + // sampled grows that set to 206 — deliberately, see servingTargetServices — + // while this row keeps promising depth on 205, and every gate stays green. + if got, want := len(servingTargetServices(t)), serving; got != want { + t.Errorf("docs/coverage.md promises depth on %d services, but the tier table's "+ + "serving-target column is counted over %d. The registry moved against "+ + "docs/demand.md: either the target is a new number and this row moves with "+ + "it, or a service left the registry and the depth promise is no longer met.", + want, got) + } } // demandPath is the evidence behind the published target. See docs/demand.md. @@ -434,6 +679,31 @@ const demandPath = "../../docs/demand.md" // | `emr-serverless` | yes | yes | yes | 3 | var demandRow = regexp.MustCompile("(?m)^\\|\\s*`([a-z0-9-]+)`\\s*\\|[^|]*\\|[^|]*\\|[^|]*\\|\\s*(\\d+)\\s*\\|") +// demandSupport reads the support column of one matched row. +// +// Both readers of docs/demand.md split on this number, in opposite directions — +// one takes support >= 2, the other takes the rest — so the two must read the +// cell the same way or a service ends up on neither side, or on both. One parse, +// used twice, is how they stay in step. +// +// The error is fatal rather than skipped. demandRow already requires digits, so +// a cell reading "n/a" drops the whole row before this is reached and only an +// overflowing number gets here; both call sites used to fold that into their +// `continue`, which read as though a malformed cell were routine. It is not: a +// row that goes missing moves a service across the published target rather than +// out of the table, which is why the two gates that notice — the pinned demand +// set and the serving-target denominator — are assertions and not filters. +func demandSupport(t *testing.T, row []string) int { + t.Helper() + + support, err := strconv.Atoi(row[2]) + if err != nil { + t.Fatalf("docs/demand.md: the support column for %q reads %q, which is not a "+ + "number this can use: %v", row[1], row[2], err) + } + return support +} + // TestDemandSetIsRegistered gates the target itself, not only the count. // // The published target is "148 registered + the 57 services with demonstrated @@ -457,8 +727,7 @@ func TestDemandSetIsRegistered(t *testing.T) { var demandSet, missing []string for _, row := range demandRow.FindAllStringSubmatch(string(raw), -1) { - support, err := strconv.Atoi(row[2]) - if err != nil || support < 2 { + if demandSupport(t, row) < 2 { continue } name := row[1] @@ -482,6 +751,60 @@ func TestDemandSetIsRegistered(t *testing.T) { } } +// servingTargetServices is the set the depth target promises: every registered +// service except the long tail the demand study found nobody building. +// +// The set is derived rather than pinned because pinning it would be a fourth +// place the 205 lives, and docs/demand.md already enumerates all 283 services +// that were missing when the study was sampled. The 226 with support < 2 are +// exactly the ones this PRD registered for routing alone; subtracting them from +// the registry leaves the 148 registered at sample time plus the 57-service +// demand set. +// +// A service AWS publishes after the study has no row here, so it joins the +// target set. That is deliberate: nobody has measured its demand either way, +// and the alternative — failing the weekly sync whenever AWS ships a service — +// buys nothing. See docs/coverage.md, which states the denominator literally. +func servingTargetServices(t *testing.T) map[string]bool { + t.Helper() + + raw, err := os.ReadFile(demandPath) + if err != nil { + t.Fatalf("read the demand evidence: %v", err) + } + + target := make(map[string]bool) + for _, id := range plugin.DefaultRegistry.RegisteredServices() { + target[id] = true + } + + var unregistered []string + for _, row := range demandRow.FindAllStringSubmatch(string(raw), -1) { + if demandSupport(t, row) >= 2 { + continue + } + // The same two vocabularies TestDemandSetIsRegistered joins: demand + // names carry the SDK's punctuation, DevCloud service IDs have none. + id := strings.ReplaceAll(row[1], "-", "") + if !target[id] { + unregistered = append(unregistered, row[1]) + continue + } + delete(target, id) + } + + // Every one of the 226 is registered — that is this PRD's whole claim. A + // name that no longer joins means either the leak-zero guarantee broke or + // the two vocabularies drifted, and both make the published share wrong + // rather than merely stale. + if len(unregistered) > 0 { + t.Fatalf("%d services the demand study found nobody building are not registered: %v. "+ + "The serving-target denominator is the registry minus exactly those, so it "+ + "cannot be derived while they are missing.", len(unregistered), unregistered) + } + return target +} + // hyphenate renders a service ID the way the docs write it: DevCloud IDs have no // punctuation, prose uses the SDK's spelling ("rds-data" for rdsdata). func hyphenate(id string) string { diff --git a/cmd/devcloud/figures_update_test.go b/cmd/devcloud/figures_update_test.go index 2e516dd..3accba7 100644 --- a/cmd/devcloud/figures_update_test.go +++ b/cmd/devcloud/figures_update_test.go @@ -46,10 +46,11 @@ type figureEdit struct { pattern *regexp.Regexp figures []figureValue - // separated writes thousands separators, as docs/coverage.md does for - // operation counts and not for service counts. Which of the two applies is - // decided by the gate's own pattern: one accepts commas, the other does not. - separated bool + // render writes the figure the way the target cell shows it. docs/coverage.md + // thousands-separates operation counts, leaves service counts plain, and + // shows the fidelity share to one decimal place — three renderings, and the + // cell decides which, so the edit that owns the cell carries it. + render func(int) string // everyMatch rewrites all matches rather than insisting on exactly one. The // front pages state the figure in a sentence, and a page is free to state it @@ -91,6 +92,7 @@ func servingTarget(t *testing.T, root string) int { func publishedFigureEdits(f figures, servingTarget int) []figureEdit { const ( coverage = "docs/coverage.md" + manifest = "docs/fidelity-manifest.md" readme = "README.md" docsIndex = "docs/README.md" ) @@ -105,32 +107,69 @@ func publishedFigureEdits(f figures, servingTarget int) []figureEdit { } } + // The share is stated on two pages now, so its rows carry the file name for + // the same reason pair() does: two rows both labelled "hand-verified share" + // read as a table that repeated itself rather than as two pages that have to + // agree. The values are tenths of a percent — see shareTenths. + shares := func(path string) []figureValue { + return []figureValue{ + {fmt.Sprintf("hand-verified share, serving target (%s)", path), + shareTenths(f.targetTiers[fidelity.TierHandVerified], f.targetTotalKnown)}, + {fmt.Sprintf("hand-verified share, all registered (%s)", path), + shareTenths(f.tiers[fidelity.TierHandVerified], f.totalKnown)}, + } + } + + tierEdit := func(tier fidelity.Tier) figureEdit { + return figureEdit{path: coverage, pattern: tierRowPattern(string(tier)), render: separatedFigure, + figures: []figureValue{ + {fmt.Sprintf("`%s` operations (serving target)", tier), f.targetTiers[tier]}, + {fmt.Sprintf("`%s` operations (all registered)", tier), f.tiers[tier]}, + }} + } + return []figureEdit{ - {path: coverage, pattern: coverageRowPattern("Registered"), + {path: coverage, pattern: coverageRowPattern("Registered"), render: plainFigure, figures: []figureValue{{"Registered", f.registered}}}, - {path: coverage, pattern: coverageRowPattern("Serving ≥1 operation"), + {path: coverage, pattern: coverageRowPattern("Serving ≥1 operation"), render: plainFigure, figures: []figureValue{{"Serving ≥1 operation", f.serving}}}, - {path: coverage, pattern: coverageRowPattern("Registered-only"), + {path: coverage, pattern: coverageRowPattern("Registered-only"), render: plainFigure, figures: []figureValue{{"Registered-only", len(f.registeredOnly)}}}, - {path: coverage, pattern: coverageRowPattern("Compatibility-tested"), + {path: coverage, pattern: coverageRowPattern("Compatibility-tested"), render: plainFigure, figures: []figureValue{{"Compatibility-tested", f.compatTested}}}, - {path: coverage, pattern: tierRowPattern(string(fidelity.TierHandVerified)), separated: true, - figures: []figureValue{{"`hand-verified` operations", f.tiers[fidelity.TierHandVerified]}}}, - {path: coverage, pattern: tierRowPattern(string(fidelity.TierAutoCRUD)), separated: true, - figures: []figureValue{{"`auto-crud` operations", f.tiers[fidelity.TierAutoCRUD]}}}, - {path: coverage, pattern: tierRowPattern(string(fidelity.TierUnimplemented)), separated: true, - figures: []figureValue{{"`unimplemented` operations", f.tiers[fidelity.TierUnimplemented]}}}, - {path: coverage, pattern: totalKnownPattern, separated: true, - figures: []figureValue{{"total known operations", f.totalKnown}}}, - - {path: coverage, pattern: targetRowPattern("Routing target"), separated: true, + tierEdit(fidelity.TierHandVerified), + tierEdit(fidelity.TierAutoCRUD), + tierEdit(fidelity.TierUnimplemented), + {path: coverage, pattern: totalKnownPattern, render: separatedFigure, + figures: []figureValue{ + {"total known operations (serving target)", f.targetTotalKnown}, + {"total known operations (all registered)", f.totalKnown}, + }}, + {path: coverage, pattern: handVerifiedSharePattern, render: formatTenths, + figures: shares(coverage)}, + + // The two figures the paragraph under that table states in words. They are + // the difference between its columns, so they move with it — and prose the + // sync leaves behind is the drift this whole tool exists to stop. + {path: coverage, pattern: longTailPattern, render: separatedFigure, + figures: []figureValue{ + {"operations outside the serving target", f.totalKnown - f.targetTotalKnown}}}, + {path: coverage, pattern: longTailHandVerifiedPattern, render: separatedFigure, + figures: []figureValue{ + {"`hand-verified` operations outside the serving target", + f.tiers[fidelity.TierHandVerified] - f.targetTiers[fidelity.TierHandVerified]}}}, + + {path: coverage, pattern: targetRowPattern("Routing target"), render: separatedFigure, figures: []figureValue{{"Routing target", f.registered}}}, - {path: coverage, pattern: targetRowPattern("Registered and engine-served, outside the serving target"), separated: true, + {path: coverage, pattern: targetRowPattern("Registered and engine-served, outside the serving target"), render: separatedFigure, figures: []figureValue{{"Outside the serving target", f.registered - servingTarget}}}, - {path: readme, pattern: quotedFigurePattern, everyMatch: true, figures: pair(readme)}, - {path: docsIndex, pattern: quotedFigurePattern, everyMatch: true, figures: pair(docsIndex)}, + {path: manifest, pattern: manifestSharePattern, render: formatTenths, + figures: shares(manifest)}, + + {path: readme, pattern: quotedFigurePattern, everyMatch: true, render: plainFigure, figures: pair(readme)}, + {path: docsIndex, pattern: quotedFigurePattern, everyMatch: true, render: plainFigure, figures: pair(docsIndex)}, } } @@ -169,7 +208,7 @@ func applyFigureEdits(doc string, edits []figureEdit) (string, []figureChange, e changes = append(changes, figureChange{ label: fv.label, before: doc[m[2*(i+1)]:m[2*(i+1)+1]], - after: formatFigure(fv.want, e.separated), + after: e.render(fv.want), }) } } @@ -180,7 +219,7 @@ func applyFigureEdits(doc string, edits []figureEdit) (string, []figureChange, e m := locs[i] for g := len(e.figures) - 1; g >= 0; g-- { start, end := m[2*(g+1)], m[2*(g+1)+1] - doc = doc[:start] + formatFigure(e.figures[g].want, e.separated) + doc[end:] + doc = doc[:start] + e.render(e.figures[g].want) + doc[end:] } } } @@ -212,6 +251,25 @@ func formatFigure(n int, separated bool) string { return sign + s } +// plainFigure and separatedFigure name the two renderings docs/coverage.md +// already used, so publishedFigureEdits reads as a list of cells rather than a +// list of booleans. +func plainFigure(n int) string { return formatFigure(n, false) } +func separatedFigure(n int) string { return formatFigure(n, true) } + +// formatTenths renders tenths of a percent as the page shows them: 362 -> "36.2". +// +// The share is carried as an integer so the gate and this tool compare the +// rendered digits rather than two floats that agree to a precision the page +// never displays. See shareTenths in coverage_test.go. +func formatTenths(n int) string { + sign := "" + if n < 0 { + sign, n = "-", -n + } + return fmt.Sprintf("%s%d.%d", sign, n/10, n%10) +} + // renderFigureTable is what a reviewer reads instead of re-deriving. // // Every figure is listed, not only the ones that moved: "these four did not @@ -422,10 +480,11 @@ func TestUpdaterRestoresAMangledFigure(t *testing.T) { t.Error("a mangled **Registered** row was not restored to the committed page") } - // 2. A tier row: the thousands-separated shape. Restoring "7" to "10,871" - // proves the separator survives the round trip rather than being dropped. + // 2. A tier row: the thousands-separated shape, both columns. Restoring + // "7"/"8" to "5,193"/"10,871" proves the separator survives the round trip + // and that the two denominators are not written into each other's cells. if got := restore("docs/coverage.md", - mangleFigure(t, coverage, tierRowPattern(string(fidelity.TierAutoCRUD)), "7")); got != coverage { + mangleFigure(t, coverage, tierRowPattern(string(fidelity.TierAutoCRUD)), "7", "8")); got != coverage { t.Error("a mangled `auto-crud` tier row was not restored, so the thousands " + "separator does not survive the rewrite") } @@ -467,6 +526,43 @@ func TestUpdaterRestoresAMangledFigure(t *testing.T) { "it moved with the numbers", rel) } } + + // 6. The share row: one decimal place, and a percent sign the updater must + // leave where it is. Only the digits are captured, so "36.2%" coming back + // as "36.2" would mean the rewrite swallowed the unit. + if got := restore("docs/coverage.md", + mangleFigure(t, coverage, handVerifiedSharePattern, "9.9", "8.8")); got != coverage { + t.Error("a mangled hand-verified share row was not restored, or the rewrite " + + "consumed the percent sign") + } + + // 7. The total row, both columns: the same shape as case 2 but bold, which is + // a different pattern and so a different splice. + if got := restore("docs/coverage.md", + mangleFigure(t, coverage, totalKnownPattern, "1", "2")); got != coverage { + t.Error("a mangled **total known** row was not restored in both denominators") + } + + // 8 and 9. The two figures the paragraph under the tier table states in a + // sentence. Splicing into prose is where a rewrite is most likely to eat a + // neighbouring word, and the whole-file comparison is what catches it. + for _, p := range []*regexp.Regexp{longTailPattern, longTailHandVerifiedPattern} { + if got := restore("docs/coverage.md", mangleFigure(t, coverage, p, "3")); got != coverage { + t.Error("a mangled long-tail figure was not restored, so the sentence around " + + "it does not survive the rewrite") + } + } + + // 10. The other page. docs/fidelity-manifest.md restates the share in prose, + // across a line break and a Markdown link — the loosest span this tool + // writes into, and the one where an over-wide match would be invisible to + // a cell-level assertion. + manifest := restore("docs/fidelity-manifest.md", read("docs/fidelity-manifest.md")) + if got := restore("docs/fidelity-manifest.md", + mangleFigure(t, manifest, manifestSharePattern, "9.9", "8.8")); got != manifest { + t.Error("a mangled share in docs/fidelity-manifest.md was not restored, or the " + + "rewrite reached past the two figures it owns") + } } // TestQuotedFigurePatternReadsThePhrasingsInUse pins what the loose pattern is @@ -538,6 +634,7 @@ func TestUpdaterLeavesUnrelatedSentencesAlone(t *testing.T) { path: "README.md", pattern: quotedFigurePattern, everyMatch: true, + render: plainFigure, figures: []figureValue{{"Registered", 431}, {"Serving ≥1 operation", 426}}, }}) if err != nil { @@ -572,6 +669,33 @@ func TestFormatFigureWritesTheFigureTheDocsShow(t *testing.T) { } } +// TestShareRendersTheFigureThePageShows pins the arithmetic behind the one +// figure on the page that is a ratio rather than a count. +// +// The two live cases are asserted by name: they are the numbers the PRD called a +// regression and a target, and truncating instead of rounding turns 23.6% into +// 23.5% — a digit the reader sees and no float tolerance would catch. +func TestShareRendersTheFigureThePageShows(t *testing.T) { + for _, tc := range []struct { + name string + hand, total int + want string + }{ + {"the serving target, as published", 4497, 12407, "36.2"}, + {"all registered services, as published", 4528, 19201, "23.6"}, + {"rounds half up rather than truncating", 1, 8, "12.5"}, + {"a surface with nothing hand-verified", 0, 100, "0.0"}, + {"a fully hand-verified surface", 100, 100, "100.0"}, + {"an empty surface does not divide by zero", 0, 0, "0.0"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := formatTenths(shareTenths(tc.hand, tc.total)); got != tc.want { + t.Errorf("share of %d/%d = %q, want %q", tc.hand, tc.total, got, tc.want) + } + }) + } +} + // TestProseRequiredNamesWhatANumberCannotCarry covers the branch that is the // whole difference between automating the arithmetic and automating the // judgement — and that TestUpdatePublishedFigures, being env-gated, never runs diff --git a/docs/coverage.md b/docs/coverage.md index 8c9bb44..9bb5cff 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -11,14 +11,30 @@ alone. | **Registered-only** | Routed, but every operation declines with a clean AWS error. | **5** | | **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **426** | -Per operation, from the [fidelity manifest](fidelity-manifest.md): +Per operation, from the [fidelity manifest](fidelity-manifest.md). Two +denominators, because [routing and depth are two targets](#the-target): -| Tier | Operations | -|---|---| -| `hand-verified` | 4,528 | -| `auto-crud` | 10,871 | -| `unimplemented` | 3,802 | -| **total known** | **19,201** | +| Tier | Serving target | All registered | +|---|---|---| +| `hand-verified` | 4,497 | 4,528 | +| `auto-crud` | 5,193 | 10,871 | +| `unimplemented` | 2,717 | 3,802 | +| **total known** | **12,407** | **19,201** | +| **hand-verified share** | **36.2%** | **23.6%** | + +The **serving target** column is the depth promise: every registered service +except the 226 the [demand study](demand.md) found nobody building. The **all +registered** column adds those 226 back, and with them 6,794 operations that +exist so an SDK call cannot leave for a billable account — not because DevCloud +promises to serve them well. + +So the two shares answer different questions, and reading the right-hand one as +a fidelity regression is the mistake this split exists to prevent. Registering +226 more services did not make any operation less faithful; it enlarged a +denominator. Of those operations, 31 are `hand-verified` — the eight +hand-written providers for the services the engine cannot classify — and that is +the whole intended depth of the long tail. Promotion out of it happens +[on request, with a use case](fidelity-manifest.md#getting-an-operation-promoted). > **Two targets, not one: routing is 431 of 431, depth is 205.** Every service > AWS publishes is registered, so no call can leave for a billable account — that @@ -356,12 +372,16 @@ make test-compat # the compatibility-tested number, over all 431 service Every figure comes from `internal/generated/fidelity/manifest_gen.go` and nothing else, so it cannot drift from what the binary serves. It cannot drift from *this -page* either — five tests compare the two, in both directions: +page* either — nine tests compare the two, in both directions: | Test | Gates | |---|---| -| `TestPublishedCoverageMatchesTheBinary` | the three service counts | -| `TestPublishedOperationTiersMatchTheManifest` | the four operation tiers | +| `TestPublishedCoverageMatchesTheBinary` | the four service counts | +| `TestPublishedOperationTiersMatchTheManifest` | the operation tiers, in both denominators | +| `TestPublishedFidelityShareMatchesTheManifest` | the `hand-verified` share of each | +| `TestPublishedLongTailProseMatchesTheManifest` | the long-tail figures the prose states in words | +| `TestFidelityManifestQuotesTheSameShare` | that [the manifest page](fidelity-manifest.md) states the same share | +| `TestPublishedTargetTableMatchesTheBinary` | the target table's arithmetic, and that its 205 is the denominator the tier table divides by | | `TestRegisteredOnlyServicesAreNamedInTheDocs` | that a service serving nothing is named here | | `TestOtherDocsQuoteTheSameFigure` | that `README.md` and `docs/README.md` agree | | `TestDemandSetIsRegistered` | that all 57 demand-set services are still registered | diff --git a/docs/fidelity-manifest.md b/docs/fidelity-manifest.md index 461f1f4..95d9dd8 100644 --- a/docs/fidelity-manifest.md +++ b/docs/fidelity-manifest.md @@ -111,6 +111,11 @@ whole package would file both as operations. - **`hand-verified` means "the provider dispatches this operation"**, not "this matches AWS byte for byte". Depth varies; `make test-compat` is the stronger signal for the services it covers. +- **A tier share needs its denominator named.** `hand-verified` is 36.2% of the + operations inside the [serving target](coverage.md#the-target) and 23.6% of + every operation DevCloud knows about, because the second figure includes the + long tail that is registered for routing alone. Both are published in + [coverage.md](coverage.md), and neither is the share "of DevCloud". - The manifest describes the **whole registered surface**, not the services enabled in your config.