feat: gate the published coverage figure, and test every registered service

README.md

@@ -23,8 +23,8 @@ Today DevCloud targets **AWS**. Our long-term goal is to support the full range
 
 ## Features
 
-- **148 AWS services registered, 117 serving at least one operation** — the other 31 are routed and decline with a clean AWS error rather than letting the call bill a real AWS account. The split is the honest form of the number; see [coverage.md](docs/coverage.md) for what it does and does not promise, and [services-matrix.md](docs/services-matrix.md) for the list
-- **boto3-compatible** — a 729-test compatibility suite runs in CI (`make test-compat`); works with most boto3 apps, and unsupported operations return a clean AWS error, never a false success
+- **205 AWS services registered, 201 serving at least one operation** — the other 4 are routed and decline with a clean AWS error rather than letting the call bill a real AWS account. The split is the honest form of the number; see [coverage.md](docs/coverage.md) for what it does and does not promise, and [services-matrix.md](docs/services-matrix.md) for the list
+- **boto3-compatible** — a 992-test compatibility suite runs in CI (`make test-compat`) and exercises every registered service; works with most boto3 apps, and unsupported operations return a clean AWS error, never a false success
 - **Cross-service integration** — CFN provisioning, DDB Streams → Lambda, EventBridge targets, S3 → Lambda
 - **Smithy-driven codegen** — auto-generate Go types, routers, and error catalogues from Smithy models
 - **Weekly auto-sync** — GitHub Actions keeps generated code up to date with upstream AWS API changes

changes/unreleased/Added-20260906-100000.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: 'CI gates the published coverage figure. `docs/coverage.md`, `README.md` and `docs/README.md` are read by `go test ./cmd/devcloud/` and compared to the live registry and fidelity manifest in both directions: removing a service without editing the published number fails, and editing the number without the code moving fails identically. The demand set in `docs/demand.md` is checked to still be registered, so the count cannot be held steady by swapping one service for another'
+time: 2026-09-06T10:00:00.000000+09:00
+custom:
+    Issue: "144"

changes/unreleased/Changed-20260906-100100.yaml

@@ -0,0 +1,5 @@
+kind: Changed
+body: 'The boto3 compatibility suite exercises every registered service. It parametrises over a generated service list (`internal/generated/compat/services.json`) instead of two hand-written name lists, so registering a service and testing it are no longer separate steps. 31 of 205 registered services had no boto3 test at all, 29 of them counted in the published coverage figure; the suite grew from 854 to 993 tests'
+time: 2026-09-06T10:01:00.000000+09:00
+custom:
+    Issue: "144"

changes/unreleased/Documentation-20260906-100200.yaml

@@ -0,0 +1,5 @@
+kind: Documentation
+body: 'Coverage docs state what the compatibility figure excludes and why. Two registered services have no boto3 client at all (`sagemakerruntimehttp2`, `transcribestreaming`), and the four Lex services are registered but reachable by no boto3 caller — all four clients sign as the contested alias `lex`, which routes nowhere by design. `README.md` and `docs/README.md` had quoted 148 registered / 117 serving since three milestones earlier'
+time: 2026-09-06T10:02:00.000000+09:00
+custom:
+    Issue: "144"

cmd/codegen/main.go

@@ -158,11 +158,21 @@ func main() {
 			os.Exit(1)
 		}
 
-		if err := writeFidelityManifest(gen, *outputDir, modelOps, protocols, providers, crudServices); err != nil {
+		// Built once and shared: the compat manifest is a projection of the
+		// fidelity data, so deriving it from anything else would let the two
+		// disagree about which operations a service serves.
+		fidelityData := codegen.BuildFidelityData(modelOps, protocols, providers, crudServices)
+
+		if err := writeFidelityManifest(gen, *outputDir, fidelityData); err != nil {
 			fmt.Fprintf(os.Stderr, "Error writing fidelity manifest: %v\n", err)
 			os.Exit(1)
 		}
 
+		if err := writeCompatManifest(*outputDir, fidelityData); err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing compat manifest: %v\n", err)
+			os.Exit(1)
+		}
+
 		if err := writeAliases(gen, *outputDir, allModels); err != nil {
 			fmt.Fprintf(os.Stderr, "Error writing alias table: %v\n", err)
 			os.Exit(1)
@@ -200,17 +210,8 @@ func writeAliases(gen *codegen.Generator, outputDir string, models []*ir.Model)
 // writeFidelityManifest emits the per-operation fidelity manifest. Like the CRUD
 // registry it is only written on a full-fleet run, since a filtered run would
 // produce a partial manifest.
-func writeFidelityManifest(
-	gen *codegen.Generator,
-	outputDir string,
-	modelOps map[string][]string,
-	protocols map[string]string,
-	providers map[string]codegen.ProviderScan,
-	crudServices []codegen.CRUDServiceData,
-) error {
-	content, err := gen.GenerateFidelityManifest(
-		codegen.BuildFidelityData(modelOps, protocols, providers, crudServices),
-	)
+func writeFidelityManifest(gen *codegen.Generator, outputDir string, data codegen.FidelityData) error {
+	content, err := gen.GenerateFidelityManifest(data)
 	if err != nil {
 		return err
 	}
@@ -221,3 +222,21 @@ func writeFidelityManifest(
 	}
 	return codegen.WriteGo(filepath.Join(dir, "manifest_gen.go"), content)
 }
+
+// writeCompatManifest emits the service list the boto3 compatibility suite
+// parametrizes over. It is JSON rather than Go because Python reads it, and it
+// lives under internal/generated so the codegen-drift CI job diffs it alongside
+// everything else derived from the models — a service added without a
+// regeneration then fails CI instead of quietly going untested.
+func writeCompatManifest(outputDir string, data codegen.FidelityData) error {
+	content, err := codegen.RenderCompatManifest(codegen.BuildCompatManifest(data))
+	if err != nil {
+		return err
+	}
+
+	dir := filepath.Join(outputDir, "compat")
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(dir, "services.json"), content, 0644)
+}

cmd/devcloud/conformance_test.go

@@ -24,13 +24,14 @@ var validProtocols = map[plugin.ProtocolType]bool{
 func TestServicePluginConformance(t *testing.T) {
 	ids := plugin.DefaultRegistry.RegisteredServices()
 
-	// Conservative floor. Broken registration wiring (a mangled imports.go,
-	// init-order regression, a dropped Register) collapses the live surface far
-	// below this, so a constant catches it without the fragility of parsing
-	// imports.go at runtime. Raise it if the real count ever nears it.
-	const minServices = 50
-	if len(ids) < minServices {
-		t.Fatalf("registered %d services, want >= %d", len(ids), minServices)
+	// The count itself is asserted exactly against the published figure in
+	// TestPublishedCoverageMatchesTheBinary, which is what catches broken
+	// registration wiring (a mangled imports.go, an init-order regression, a
+	// dropped Register). This only needs the surface to be non-empty before
+	// iterating it; a conservative floor of 50 here would be a second, weaker
+	// answer to a question that now has an exact one.
+	if len(ids) == 0 {
+		t.Fatal("no services registered; check cmd/devcloud/imports.go")
 	}
 
 	var prev string