feat: serve rest-json from the generic CRUD engine

changes/unreleased/Added-20260905-184000.yaml

@@ -0,0 +1,5 @@
+kind: Added
+body: 'The generic CRUD engine now serves `rest-json` services. That protocol carries no `X-Amz-Target` header, so the operation is recovered from the request method and path using the route table the model already declares — the same matcher the generated per-service routers use, now shared in `internal/shared/httproute`. 28 already-registered services stopped being registered-only without any change to their providers: services serving at least one operation went 117 to 145, and `auto-crud` operations 1,415 to 2,200. A path the route table does not know still declines with a clean AWS error rather than a fabricated success'
+time: 2026-09-05T18:40:00.000000+09:00
+custom:
+    Issue: "139"

cmd/devcloud/fidelity_test.go

@@ -103,6 +103,16 @@ func TestFidelityManifestCoversRegisteredServices(t *testing.T) {
 // lists.
 func TestFidelityManifestCoversCRUDRegistry(t *testing.T) {
 	for _, id := range plugin.DefaultRegistry.RegisteredServices() {
+		// Registry membership is classifiability, not reachability. The CRUD
+		// registry is built from the model, so it also holds operations for
+		// services whose hand-written provider refuses unknown operations
+		// itself (apigatewayv2, xray) — the engine is never routed to for
+		// those, so "unimplemented" is the truth and this check would be
+		// asserting the opposite. TestFidelityManifestCoverage's `served == 0
+		// && RegisteredOps > 0` guard is what catches wiring that goes missing.
+		if !fidelity.Services[id].EngineWired {
+			continue
+		}
 		for op := range crud.RegisteredOps(id) {
 			tier, ok := fidelity.Lookup(id, op)
 			if !ok {
@@ -129,7 +139,13 @@ func TestAutoCRUDIsServedOverJSON(t *testing.T) {
 			if tier != fidelity.TierAutoCRUD {
 				continue
 			}
-			if _, err := crud.Handle(id, op, "json-1.1", []byte("{}")); err != nil {
+			// Asked through the JSON path regardless of the service's real
+			// protocol: what "auto-crud" claims is that the operation is
+			// registered and CRUD-classified, which is exactly what this
+			// exercises. Whether a rest-json request also resolves to it is
+			// route matching, covered in internal/shared/crud and the compat
+			// suite.
+			if _, err := crud.Handle(crud.Call{Service: id, Protocol: "json-1.1", Op: op, Body: []byte("{}")}); err != nil {
 				t.Errorf("%s/%s: declared auto-crud but the engine refused it: %v", id, op, err)
 			}
 		}

docs/coverage.md

@@ -10,37 +10,58 @@ things:
 | Number | What it means | Today |
 |---|---|---|
 | **Registered** | The gateway routes the service. A call reaches DevCloud instead of falling through to real AWS. | **148** |
-| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer — hand-written or served by the generic CRUD engine. | **117** |
-| **Registered-only** | Routed, but every operation declines with a clean AWS error. Nothing is served. | **31** |
+| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer — hand-written or served by the generic CRUD engine. | **145** |
+| **Registered-only** | Routed, but every operation declines with a clean AWS error. Nothing is served. | **3** |
 
 Per operation, from the [fidelity manifest](fidelity-manifest.md):
 
 | Tier | Operations |
 |---|---|
 | `hand-verified` | 4,496 |
-| `auto-crud` | 1,415 |
-| `unimplemented` | 3,119 |
+| `auto-crud` | 2,200 |
+| `unimplemented` | 2,334 |
 | **total known** | **9,030** |
 
 ## Why a registered service can serve nothing
 
-The generic CRUD engine classifies an operation by its name, and only the
-`X-Amz-Target` JSON protocols put the operation name where the router can see it
-(`internal/shared/crud/crud.go`, `JSONProtocol`). A `rest-json`, `query` or
-`rest-xml` service is therefore unreachable by the engine and serves nothing
-until somebody writes its provider by hand.
+The generic CRUD engine has to know which operation a request is for, and it has
+to recognise that operation as CRUD-shaped. Two things can stop it.
+
+**The protocol does not say which operation.** The `X-Amz-Target` JSON protocols
+name the operation in a header. `rest-json` does not, but every one of its
+operations is bound to an HTTP method and a URI template in the model, and
+`internal/shared/httproute` matches a request back to the operation from that
+pair — so the engine serves it too. `query` and `rest-xml` have neither a target
+header nor a modelled path the engine can match, so they are unreachable by it
+and serve nothing until somebody writes the provider by hand.
+
+**The operation is not CRUD-shaped.** `GetThing`, `ListThings`, `CreateThing`
+and their siblings map onto a generic store. `ExecuteStatement`, `InvokeEndpoint`
+and `QueryForecast` do not, and the engine refuses them rather than inventing an
+answer. A service whose entire API is that shape serves nothing whatever its
+protocol.
 
 Registered services by protocol:
 
 | Protocol | Services | Engine-servable |
 |---|---|---|
-| `json-1.1` | 49 | yes |
-| `json-1.0` | 11 | yes |
-| `rest-json` | 59 | no |
+| `json-1.1` | 49 | yes — operation from `X-Amz-Target` |
+| `json-1.0` | 11 | yes — operation from `X-Amz-Target` |
+| `rest-json` | 59 | yes — operation from method + path |
 | `query` | 14 | no |
 | `rest-xml` | 3 | no |
 | no in-tree model | 12 | n/a — hand-written providers |
 
+The three services that are registered and serve nothing are all the second
+case, not the first: `forecastquery` (`json-1.1`, only `QueryForecast` and
+`QueryWhatIfForecast`) and the two SageMaker Runtime variants (`rest-json`, only
+`InvokeEndpoint*`).
+
+A registered operation is not automatically a reachable one. The engine is
+entered only when a provider returns `plugin.ErrUnhandledOp`, so a hand-written
+provider that refuses unknown operations itself — `apigatewayv2`, `xray` — never
+reaches it, and the manifest records that per service as `EngineWired`.
+
 Registering a service the engine cannot serve is deliberate, not an oversight.
 The alternative is worse: an unregistered service is not routed, so the SDK call
 leaves the machine and bills a real AWS account. A registered-only service
@@ -56,19 +77,17 @@ The first category taken to completion. All 48 upstream models are registered.
 | | Count |
 |---|---|
 | Registered | 48 |
-| Engine-served or hand-written | 17 |
-| Registered-only | 31 |
-| Operations served across the category | 935 |
-
-Engine-served: `bedrock`, `bedrock-data-automation-runtime`, `comprehend`,
-`comprehendmedical`, `forecast`, `frauddetector`, `healthlake`, `kendra`,
-`kendra-ranking`, `lookoutequipment`, `personalize`, `rekognition`, `sagemaker`,
-`textract`, `transcribe`, `translate`, `voice-id`.
-
-Of the 31 registered-only, 30 are `rest-json`. The exception is `forecastquery`,
-which is `json-1.1` but whose entire API is `QueryForecast` and
-`QueryWhatIfForecast` — neither is CRUD-shaped, so there is nothing for the
-engine to serve generically.
+| Engine-served or hand-written | 45 |
+| Registered-only | 3 |
+
+When this category was completed, only 17 of its 48 members served anything: 30
+of the other 31 were `rest-json`, which the engine could not read at all. Teaching
+it that protocol moved 28 of those 30 into coverage without touching a single one
+of their providers.
+
+The three that remain are the ones no protocol change can help — `forecastquery`
+and the two SageMaker Runtime variants, none of which has a CRUD-shaped
+operation. See [Why a registered service can serve nothing](#why-a-registered-service-can-serve-nothing).
 
 ## What counts as a service
 

internal/codegen/gen_crud_meta.go

@@ -28,6 +28,12 @@ type crudOpData struct {
 	Resource string
 	ListKey  string
 	ItemKey  string
+	// Method and URI are the operation's REST binding, empty for a protocol
+	// that has none. rest-json names its operation in the method and path
+	// rather than in a header, so this pair is what lets the engine get back
+	// from a request to an operation name.
+	Method string
+	URI    string
 }
 
 // verbPrefixes maps operation-name prefixes to canonical CRUD verbs, longest and
@@ -117,21 +123,34 @@ func classifyOps(model *ir.Model) []crudOpData {
 			Resource: resource,
 			ListKey:  listKey,
 			ItemKey:  itemKey,
+			Method:   op.HTTPMethod,
+			URI:      op.HTTPUri,
 		})
 	}
 	return ops
 }
 
-// isJSONProtocol reports whether the engine can serve a service's protocol.
-// Only X-Amz-Target JSON protocols carry an operation name at the router.
-func isJSONProtocol(protocol string) bool {
-	return strings.HasPrefix(protocol, "json")
+// engineServable reports whether the engine can serve a service's protocol.
+//
+// The JSON protocols carry the operation name in X-Amz-Target. rest-json does
+// not, but every one of its operations is bound to a method and URI template,
+// and internal/shared/httproute turns that pair back into an operation name —
+// so the engine can classify it too. query and rest-xml have neither, which is
+// what PRD Milestone 5 owns; admitting them here would mean guessing.
+//
+// This is deliberately the same question crud.Servable answers at runtime. The
+// two must agree: a protocol classified here but refused there registers
+// operations nothing can reach, and the fidelity manifest would call them
+// auto-crud.
+func engineServable(protocol string) bool {
+	return strings.HasPrefix(protocol, "json") || protocol == "rest-json"
 }
 
-// ServiceCRUDData classifies a JSON-protocol model's CRUD operations. It returns
-// (data, false) when the service is not engine-servable or has no CRUD ops.
+// ServiceCRUDData classifies an engine-servable model's CRUD operations. It
+// returns (data, false) when the service's protocol cannot be served or the
+// model has no CRUD-shaped operation at all.
 func ServiceCRUDData(model *ir.Model) (CRUDServiceData, bool) {
-	if !isJSONProtocol(model.Protocol) {
+	if !engineServable(model.Protocol) {
 		return CRUDServiceData{}, false
 	}
 	ops := classifyOps(model)

internal/codegen/gen_crud_meta_test.go

@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: Apache-2.0
+
+// internal/codegen/gen_crud_meta_test.go
+package codegen
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/skyoo2003/devcloud/internal/codegen/ir"
+)
+
+// crudModel builds a minimal model with one list-returning and one
+// item-returning output shape, so classifyOps has both output keys to find.
+func crudModel(protocol string, ops ...ir.Operation) *ir.Model {
+	return &ir.Model{
+		ServiceID:  "testsvc",
+		Protocol:   protocol,
+		Operations: ops,
+		Shapes: map[string]*ir.Shape{
+			"GraphList": {Name: "GraphList", Type: ir.ShapeList},
+			"Graph":     {Name: "Graph", Type: ir.ShapeStructure},
+			"ListGraphsOutput": {Name: "ListGraphsOutput", Type: ir.ShapeStructure, Members: []ir.Member{
+				{Name: "Graphs", TargetName: "GraphList"},
+			}},
+			"GetGraphOutput": {Name: "GetGraphOutput", Type: ir.ShapeStructure, Members: []ir.Member{
+				{Name: "Graph", TargetName: "Graph"},
+			}},
+		},
+	}
+}
+
+func opsOf(data CRUDServiceData) map[string]crudOpData {
+	out := make(map[string]crudOpData, len(data.Ops))
+	for _, op := range data.Ops {
+		out[op.Op] = op
+	}
+	return out
+}
+
+// TestServiceCRUDDataAcceptsRESTJSON is the change that makes Milestone 4
+// possible: 33 of the 57 demand-set services are restJson1, and refusing the
+// protocol here is what left them registered but serving nothing.
+func TestServiceCRUDDataAcceptsRESTJSON(t *testing.T) {
+	model := crudModel("rest-json",
+		ir.Operation{Name: "ListGraphs", OutputName: "ListGraphsOutput",
+			HTTPMethod: "GET", HTTPUri: "/v1/graphs"},
+		ir.Operation{Name: "GetGraph", OutputName: "GetGraphOutput",
+			HTTPMethod: "GET", HTTPUri: "/v1/graphs/{GraphName}"},
+	)
+
+	data, ok := ServiceCRUDData(model)
+	require.True(t, ok, "rest-json service must be engine-servable")
+
+	ops := opsOf(data)
+	require.Contains(t, ops, "ListGraphs")
+	require.Contains(t, ops, "GetGraph")
+
+	// Without the REST binding the engine has no way back from a request to an
+	// operation name, so carrying it is the whole point.
+	assert.Equal(t, "GET", ops["ListGraphs"].Method)
+	assert.Equal(t, "/v1/graphs", ops["ListGraphs"].URI)
+	assert.Equal(t, "/v1/graphs/{GraphName}", ops["GetGraph"].URI)
+
+	// Classification itself is unchanged: the verb still comes from the name.
+	assert.Equal(t, "List", ops["ListGraphs"].Verb)
+	assert.Equal(t, "Graphs", ops["ListGraphs"].ListKey)
+	assert.Equal(t, "Get", ops["GetGraph"].Verb)
+	assert.Equal(t, "Graph", ops["GetGraph"].ItemKey)
+}
+
+// TestServiceCRUDDataJSONUnchanged pins that admitting rest-json did not alter
+// what the JSON protocols produce. A JSON model has no HTTP binding, and
+// emitting an empty URI must not create a route that matches everything.
+func TestServiceCRUDDataJSONUnchanged(t *testing.T) {
+	for _, protocol := range []string{"json-1.0", "json-1.1"} {
+		t.Run(protocol, func(t *testing.T) {
+			model := crudModel(protocol,
+				ir.Operation{Name: "ListGraphs", OutputName: "ListGraphsOutput"},
+			)
+
+			data, ok := ServiceCRUDData(model)
+			require.True(t, ok)
+
+			ops := opsOf(data)
+			assert.Equal(t, "List", ops["ListGraphs"].Verb)
+			assert.Empty(t, ops["ListGraphs"].Method)
+			assert.Empty(t, ops["ListGraphs"].URI)
+		})
+	}
+}
+
+// TestServiceCRUDDataRejectsUnservableProtocols holds the boundary Milestone 5
+// owns. query and rest-xml put the operation neither in a header nor in a
+// modelled path the engine can match, so admitting them would serve fabricated
+// answers to callers the engine cannot actually understand.
+func TestServiceCRUDDataRejectsUnservableProtocols(t *testing.T) {
+	for _, protocol := range []string{"query", "rest-xml"} {
+		t.Run(protocol, func(t *testing.T) {
+			model := crudModel(protocol,
+				ir.Operation{Name: "ListGraphs", OutputName: "ListGraphsOutput",
+					HTTPMethod: "GET", HTTPUri: "/v1/graphs"},
+			)
+
+			_, ok := ServiceCRUDData(model)
+			assert.False(t, ok, "%s must not be engine-servable", protocol)
+		})
+	}
+}
+
+// TestServiceCRUDDataSkipsUnclassifiableService is the rds-data case: a
+// rest-json service whose entire API is ExecuteStatement-shaped classifies
+// nothing, so it registers nothing and routes nothing, and every call to it
+// gets a clean error instead of an invented success.
+func TestServiceCRUDDataSkipsUnclassifiableService(t *testing.T) {
+	model := crudModel("rest-json",
+		ir.Operation{Name: "ExecuteStatement", OutputName: "GetGraphOutput",
+			HTTPMethod: "POST", HTTPUri: "/Execute"},
+		ir.Operation{Name: "BeginTransaction", OutputName: "GetGraphOutput",
+			HTTPMethod: "POST", HTTPUri: "/BeginTransaction"},
+	)
+
+	_, ok := ServiceCRUDData(model)
+	assert.False(t, ok, "a service with no CRUD-shaped operation must register nothing")
+}

internal/codegen/gen_fidelity.go

@@ -24,6 +24,13 @@ type FidelityServiceData struct {
 	// only this field lets a reader tell them apart.
 	Protocol    string
 	ModelBacked bool
+	// EngineWired mirrors ProviderScan.EngineWired: does this provider hand
+	// unimplemented operations to the CRUD engine? Published because the CRUD
+	// registry is model-derived and so holds operations for providers the
+	// engine is never reached from; without this, "registered in the engine"
+	// reads as "served", which is the overstatement the manifest exists to
+	// prevent.
+	EngineWired bool
 	Ops         []FidelityOpData
 }
 
@@ -109,6 +116,7 @@ func BuildFidelityData(
 			ServiceID:   serviceID,
 			Protocol:    protocols[serviceID],
 			ModelBacked: len(modelOps[serviceID]) > 0,
+			EngineWired: provider.EngineWired,
 			Ops:         ops,
 		})
 	}

internal/codegen/gen_router_test.go

@@ -15,11 +15,15 @@ func TestGenerateRouter(t *testing.T) {
 	output, err := gen.GenerateRouter("s3", model)
 	require.NoError(t, err)
 	assert.Contains(t, output, "package s3")
-	assert.Contains(t, output, "type OperationRoute struct")
 	assert.Contains(t, output, "OperationRoutes")
 	assert.Contains(t, output, `"CreateBucket"`)
 	assert.Contains(t, output, `"PUT"`)
 	assert.Contains(t, output, `"/{Bucket}"`)
+	// The route table is generated per service; the matching is not. Asserting
+	// the delegation is what stops a future edit from re-inlining the algorithm
+	// into 148 packages, which is the drift internal/shared/httproute exists to
+	// prevent.
+	assert.Contains(t, output, "httproute.Match(OperationRoutes, method, uri)")
 }
 
 func TestGenerateErrors(t *testing.T) {

internal/codegen/templates/crud_registry.go.tmpl

@@ -12,7 +12,7 @@ func init() {
 {{- range .Services }}
 	crud.Register("{{ .ServiceID }}", map[string]crud.OpMeta{
 {{- range .Ops }}
-		"{{ .Op }}": {Verb: "{{ .Verb }}", Resource: "{{ .Resource }}", OutputListKey: "{{ .ListKey }}", OutputItemKey: "{{ .ItemKey }}"},
+		"{{ .Op }}": {Verb: "{{ .Verb }}", Resource: "{{ .Resource }}", OutputListKey: "{{ .ListKey }}", OutputItemKey: "{{ .ItemKey }}"{{ if .URI }}, Method: "{{ .Method }}", URI: "{{ .URI }}"{{ end }}},
 {{- end }}
 	})
 {{- end }}

internal/codegen/templates/fidelity_manifest.go.tmpl

@@ -46,13 +46,23 @@ type Service struct {
 	// service's API. When false the unimplemented tail is not knowable, so the
 	// manifest lists only what DevCloud serves.
 	ModelBacked bool
+	// EngineWired reports whether the provider hands the operations it does not
+	// implement to the generic CRUD engine, by returning plugin.ErrUnhandledOp.
+	//
+	// It is the difference between an operation being *classifiable* and being
+	// *reachable*. The CRUD registry is built from the model, so it holds
+	// entries for services whose hand-written provider refuses unknown
+	// operations itself and never routes to the engine — for those, a
+	// registered operation is genuinely unimplemented, and only this field says
+	// so without guessing.
+	EngineWired bool
 	Operations  map[string]Tier
 }
 
 // Services maps a registered service ID to its operations.
 var Services = map[string]Service{
 {{- range .Services }}
-	"{{ .ServiceID }}": {Protocol: "{{ .Protocol }}", ModelBacked: {{ .ModelBacked }}, Operations: map[string]Tier{
+	"{{ .ServiceID }}": {Protocol: "{{ .Protocol }}", ModelBacked: {{ .ModelBacked }}, EngineWired: {{ .EngineWired }}, Operations: map[string]Tier{
 {{- range .Ops }}
 		"{{ .Name }}": Tier{{ .TierConst }},
 {{- end }}