diff --git a/.github/ISSUE_TEMPLATE/service_request.yml b/.github/ISSUE_TEMPLATE/service_request.yml new file mode 100644 index 00000000..23ee45df --- /dev/null +++ b/.github/ISSUE_TEMPLATE/service_request.yml @@ -0,0 +1,67 @@ +name: Service Not Supported +description: Ask for an AWS service DevCloud does not register yet +title: "[Service] " +labels: ["enhancement", "services"] +body: + - type: markdown + attributes: + value: | + DevCloud registers a subset of the AWS services AWS publishes. When a call + reaches a service that is not registered it returns `UnknownService` + rather than reaching a provider. + + This form is the demand signal that decides which services get added next. + See [docs/coverage.md](https://github.com/skyoo2003/devcloud/blob/main/docs/coverage.md) + for what is registered today, and what "registered" does and does not promise. + + - type: input + id: service + attributes: + label: Service + description: The AWS service name, as your SDK or tool spells it. + placeholder: e.g. appflow, amp, secretsmanager + validations: + required: true + + - type: input + id: client + attributes: + label: SDK or tool that hit the wall + description: What was making the call, and which version. + placeholder: e.g. boto3 1.35.0 / Terraform aws provider 5.60 / AWS CLI v2 + validations: + required: true + + - type: textarea + id: blocked + attributes: + label: What did it block? + description: > + What you could not do because the service was missing — a test that had to + be skipped, a `terraform apply` that failed, a CI job pointed at real AWS. + Which operations you actually call matters more than the service alone. + placeholder: | + Our integration suite creates an AppFlow flow in setup, so the whole + module is skipped locally and only runs in the billed CI account. + Operations we use: CreateFlow, DescribeFlow, DeleteFlow. + validations: + required: true + + - type: textarea + id: unrouted + attributes: + label: Unrouted-service output (optional) + description: > + With `admin.enabled: true`, DevCloud counts the services it was asked for + and does not register. Paste the output of + `curl -s localhost:4747/devcloud/api/unrouted` — it turns a request into a + measurement. + + + Two caveats before you read it: a request DevCloud cannot classify at all + is routed to S3 rather than counted here, so the list is a floor and not a + census; and a non-zero `droppedServiceIds` means the collector hit its cap + and the list is incomplete. + render: json + validations: + required: false diff --git a/README.md b/README.md index 68051d97..76226fcf 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Today DevCloud targets **AWS**. Our long-term goal is to support the full range ## Features -- **104 AWS services** across all major categories (run `make stats` for current counts; see [services-matrix.md](docs/services-matrix.md)) +- **148 AWS services registered, 117 serving at least one operation** — the other 31 are routed and decline with a clean AWS error rather than letting the call bill a real AWS account. The split is the honest form of the number; see [coverage.md](docs/coverage.md) for what it does and does not promise, and [services-matrix.md](docs/services-matrix.md) for the list - **boto3-compatible** — a 729-test compatibility suite runs in CI (`make test-compat`); works with most boto3 apps, and unsupported operations return a clean AWS error, never a false success - **Cross-service integration** — CFN provisioning, DDB Streams → Lambda, EventBridge targets, S3 → Lambda - **Smithy-driven codegen** — auto-generate Go types, routers, and error catalogues from Smithy models diff --git a/changes/unreleased/Added-20260905-180000.yaml b/changes/unreleased/Added-20260905-180000.yaml new file mode 100644 index 00000000..7f57cf51 --- /dev/null +++ b/changes/unreleased/Added-20260905-180000.yaml @@ -0,0 +1,5 @@ +kind: Added +body: 'Admin API endpoint `GET /devcloud/api/unrouted` reports which AWS services this DevCloud was asked for and does not register, with per-service counts and first/last seen. Requires `admin.enabled: true`. Two documented ceilings: a request DevCloud cannot classify is routed to S3 rather than counted, so the figure is a floor; and a non-zero `droppedServiceIds` means the collector hit its key cap. Paste the output into a service request to turn a missing service into a measurement' +time: 2026-09-05T18:00:00.000000+09:00 +custom: + Issue: "138" diff --git a/changes/unreleased/Added-20260905-180001.yaml b/changes/unreleased/Added-20260905-180001.yaml new file mode 100644 index 00000000..8d414334 --- /dev/null +++ b/changes/unreleased/Added-20260905-180001.yaml @@ -0,0 +1,5 @@ +kind: Added +body: '"Service Not Supported" issue form for asking for an AWS service DevCloud does not register yet. It asks which operations you actually call and accepts the `/devcloud/api/unrouted` output, because a service name alone does not say how much of it you need' +time: 2026-09-05T18:00:01.000000+09:00 +custom: + Issue: "138" diff --git a/changes/unreleased/Changed-20260905-180002.yaml b/changes/unreleased/Changed-20260905-180002.yaml new file mode 100644 index 00000000..a807c0bc --- /dev/null +++ b/changes/unreleased/Changed-20260905-180002.yaml @@ -0,0 +1,5 @@ +kind: Changed +body: 'The AWS coverage target is 205 services, not 431. Of the 283 services DevCloud does not register, only 57 have been built by at least two of moto, LocalStack, and terraform-provider-aws, and 115 by none of them; DevCloud has received zero service requests to date. Rather than build a long tail nobody has asked for, the target is now every registered service plus every service with demonstrated demand. The remaining 226 are not refused — any of them can be onboarded on request — but they are no longer promised. Evidence and method in docs/demand.md; the decision rule was fixed in advance and is published in docs/coverage.md' +time: 2026-09-05T18:00:02.000000+09:00 +custom: + Issue: "138" diff --git a/changes/unreleased/Fixed-20260905-180003.yaml b/changes/unreleased/Fixed-20260905-180003.yaml new file mode 100644 index 00000000..b2be4e3a --- /dev/null +++ b/changes/unreleased/Fixed-20260905-180003.yaml @@ -0,0 +1,5 @@ +kind: Fixed +body: 'Stale service counts across the docs. README.md advertised 104 services and docs/faq.md 101, when 148 are registered; docs/fidelity-manifest.md also carried outdated per-tier operation totals. Every count now appears with its depth split (148 registered / 117 serving at least one operation), so the number is never read as a capability claim on its own' +time: 2026-09-05T18:00:03.000000+09:00 +custom: + Issue: "138" diff --git a/cmd/devcloud/main.go b/cmd/devcloud/main.go index 9af3f088..0e03aee7 100644 --- a/cmd/devcloud/main.go +++ b/cmd/devcloud/main.go @@ -120,14 +120,22 @@ func main() { // The log collector is only built when admin is enabled; otherwise no // consumer can ever read it, so a nil collector keeps Add off the request // hot path (see gateway.New). + // unroutedCollector follows the same rule and answers a different question: + // which services was this DevCloud asked for that it does not register. Its + // cap has headroom over the upstream model count — a real deployment cannot + // name more distinct AWS services than AWS publishes, so reaching it means + // something is sending junk, and the report says so rather than truncating + // in silence. var logCollector *admin.LogCollector + var unroutedCollector *admin.UnroutedCollector adminHandler := http.NotFoundHandler() if cfg.Admin.Enabled { logCollector = admin.NewLogCollector(1000) - adminHandler = admin.NewAPI(registry, logCollector).Handler() + unroutedCollector = admin.NewUnroutedCollector(1000) + adminHandler = admin.NewAPI(registry, logCollector, unroutedCollector).Handler() slog.Info("admin API enabled") } - gw := gateway.New(cfg.Server.Port, registry, adminHandler, logCollector) + gw := gateway.New(cfg.Server.Port, registry, adminHandler, logCollector, unroutedCollector) sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) diff --git a/docs/README.md b/docs/README.md index 9a723a19..f10b5d55 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,9 @@ This directory holds DevCloud's technical documentation. Use the map below to ju - **[Configuration](configuration.md)** — YAML options, env-var overrides, tier shortcuts - **[Architecture](architecture.md)** — system design, plugin model, codegen pipeline, multi-CSP vision - **[Roadmap](roadmap.md)** — phased plan toward multi-CSP support -- **[Services Matrix](services-matrix.md)** — 104 services, coverage status, boto3 pass rate +- **[Coverage](coverage.md)** — 148 registered / 117 serving, what the count promises, and the target +- **[Demand](demand.md)** — which unregistered services have demonstrated demand, and how that was measured +- **[Services Matrix](services-matrix.md)** — per-service coverage status and boto3 pass rate ## What you can rely on diff --git a/docs/compatibility-policy.md b/docs/compatibility-policy.md index b5d66abb..066fc6ac 100644 --- a/docs/compatibility-policy.md +++ b/docs/compatibility-policy.md @@ -120,8 +120,9 @@ Depending on any of the following will break, and breaking it is **not** a major migration. Do not treat DevCloud as a database. - **`unimplemented` → served transitions.** An operation that returns an error today may start returning a response. This is additive, and ships in a minor release. -- **Service coverage.** New services may be added in a minor release. The 104 services present - at v1.0 are a floor, not a ceiling. +- **Service coverage.** New services may be added in a minor release. The 148 services registered + today are a floor, not a ceiling — and not a promise of depth either: 31 of them serve no + operation and only decline cleanly. See [coverage.md](coverage.md). - **Error codes, HTTP status and message wording.** What *is* guaranteed for an `unimplemented` operation is that it **fails** — an AWS-shaped error, never a fabricated success. Which error is not: it comes from whichever provider handles the request, so it is `InvalidAction` (400) diff --git a/docs/coverage.md b/docs/coverage.md index 3d5ed99c..548091eb 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -4,6 +4,9 @@ DevCloud's service count is not a capability claim on its own, so this page neve states it alone. Three numbers describe the surface, and they mean different things: +> **The coverage target is not 100% of AWS.** It was, and the evidence did not +> support it — see [The target, and the rule that set it](#the-target-and-the-rule-that-set-it). + | Number | What it means | Today | |---|---|---| | **Registered** | The gateway routes the service. A call reaches DevCloud instead of falling through to real AWS. | **148** | @@ -88,6 +91,103 @@ left unrouted rather than guessed: all four Lex services sign as `lex` and none is called `lex`, so `lex` routes nowhere and each Lex service is reached by its own unambiguous name. +## The target, and the rule that set it + +**Decided 2026-09-05. The target is 205 services, not 431.** + +DevCloud's roadmap previously aimed at every AWS service AWS publishes — 431 +model files. That target rested on an assumption nobody had tested: that the +services DevCloud does not register are services anyone wants. Before committing +to building ~283 of them, the assumption was tested, and it did not hold. + +**The rule was fixed before the numbers were seen.** Four outcomes were written +down in advance, including one for "the method itself failed", specifically so +the result could not be argued into whichever answer was most convenient once it +arrived. The full evidence is in [demand.md](demand.md), re-derivable with +`python3 scripts/demand_rank.py`. + +**What was measured.** For each of the 283 unregistered services, how many of +three independent projects have already built it: `moto` (163 services), +LocalStack (119), `terraform-provider-aws` (273). Each serves a population +DevCloud names as its user, and each only adds a service when someone asks. + +| Reading | Value | +|---|---| +| Missing services (`M`) | 283 | +| `M` built by all three projects | 8 | +| `M` built by at least two | **57 (20.1%)** | +| `M` built by exactly one | 111 | +| `M` built by none | 115 | +| DevCloud's own service requests, all time | **0** | + +**The outcome that fired.** The rule kept the 100% target only if ≥60% of `M` +had support ≥ 2, and narrowed to a demand set if ≥100 did. 57 cleared neither +bar, so the pre-registered consequence applies: **the 100% claim is dropped from +the docs before any bulk work, and the published target becomes the demand set.** + +Four fifths of the AWS surface DevCloud does not cover is surface that three +projects — with far more history and staffing than DevCloud — have collectively +declined to build. That is not an accident of their roadmaps. It is what a long +tail looks like. + +| | Services | +|---|---| +| Registered today | 148 | +| Target: registered + demonstrated demand | **205** | +| Explicitly not targeted | 226 | + +The 226 are not refused. Any of them can be onboarded when someone asks — the +["Service not supported"](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml) +form is that channel, and one report outranks all three proxies, because it is +demand rather than a stand-in for it. What changed is that they are no longer +work DevCloud has promised. + +**What this verdict is not.** The three sources measure *emulator and provider +effort*, not user demand. They are a proxy, chosen because DevCloud had no +measurement of its own and the alternative was an unbounded wait. The instrument +below now accrues the real thing. + +## Service not supported? + +DevCloud counts what it was asked for and could not route. Enable the admin API +(`admin.enabled: true`) and read: + +```bash +curl -s localhost:4747/devcloud/api/unrouted | jq . +``` + +```json +{ + "services": [ + { "serviceId": "appflow", "count": 2, + "firstSeen": "2026-09-05T17:54:25+09:00", + "lastSeen": "2026-09-05T17:54:40+09:00" } + ], + "maxServiceIds": 1000, + "droppedServiceIds": 0 +} +``` + +Then open a [service request](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml) +and paste it. That output is what moves a service from the 226 into the target. + +**Two ceilings, so the number is not read as more than it is.** + +1. **It is a floor, not a census.** `gateway.DetectProtocol` classifies a request + it cannot identify as `("rest-xml", "s3")`, and S3 is registered — so an + unrecognisable request is routed to S3 rather than counted as a miss. What + this does catch is the case that matters: a real SDK or CLI call to an + unregistered service signs with that service's own name and misses the + registry. Verified with boto3 — `boto3.client("appflow")` is recorded as + `appflow`. +2. **A non-zero `droppedServiceIds` means the list is incomplete.** Service IDs + come from caller-controlled headers, so the collector caps how many distinct + ones it holds and reports both the cap and what it dropped, rather than + growing without bound or truncating in silence. + +The counts live in memory and reset when the process does. This is a local +development tool; nothing is sent anywhere. + ## Runtime cost Measured on an Apple Silicon machine, `CGO_ENABLED=0`, going from 105 to 147 diff --git a/docs/demand.md b/docs/demand.md new file mode 100644 index 00000000..bad4a504 --- /dev/null +++ b/docs/demand.md @@ -0,0 +1,339 @@ +# Demand for unregistered services + +Sampled 2026-09-05. Re-derive with `python3 scripts/demand_rank.py`. + +## What this measures, and what it does not + +DevCloud has no usage telemetry, so this page does **not** measure what +DevCloud's users ask for. It measures *revealed* demand: which of the AWS +services DevCloud does not register have already been built by three +independent projects serving the same populations DevCloud names as its +users. + +| Source | What it is | Services | +|---|---|---| +| [moto](https://github.com/getmoto/moto) | Python AWS mocking library; the same population as DevCloud's primary user, boto3 developers running tests | 163 | +| [LocalStack](https://docs.localstack.cloud/references/coverage/) | Local AWS emulator; community and pro tiers combined | 119 | +| [terraform-provider-aws](https://github.com/hashicorp/terraform-provider-aws) | One Go package per AWS service; the IaC population | 273 | + +Each of those projects adds a service because somebody asked for it. None +of them measured DevCloud's users. Read a high support count as *this +service is worth emulating to someone*, not as *our users want this*. + +The live measurement of DevCloud's own traffic is +`GET /devcloud/api/unrouted`, which accrues behind this page. Its ceiling +is documented in [coverage.md](coverage.md). + +## Readings + +| Reading | Value | +|---|---| +| Upstream model files | 431 | +| Registered by DevCloud | 148 | +| **R1 — missing set `M`** | **283** | +| R2 — `M` with support 3 | 8 | +| R2 — `M` with support 2 | 49 | +| R2 — `M` with support 1 | 111 | +| R2 — `M` with support 0 | 115 | +| **R2 — `M` with support ≥ 2** | **57 (20.1% of `M`)** | + +Join diagnostics — a name a source publishes that matches neither `M` nor a +registered DevCloud service. A large number here means the name +normalisation is dropping matches and the support counts are too low: + +| Source | Unmatched names | +|---|---| +| moto | 0 | +| LocalStack | 4 | +| terraform-provider-aws | 4 | + +## Ranking + +Ordered by support count, then name. This is the order Milestone 4 works +in, so stopping at any point stops on the best surface available. + +| Service | moto | LocalStack | terraform-provider-aws | Support | +|---|---|---|---|---| +| `api-gateway` | yes | yes | yes | 3 | +| `elastic-load-balancing` | yes | yes | yes | 3 | +| `emr-serverless` | yes | yes | yes | 3 | +| `mediastore` | yes | yes | yes | 3 | +| `rds-data` | yes | yes | yes | 3 | +| `redshift-data` | yes | yes | yes | 3 | +| `s3-control` | yes | yes | yes | 3 | +| `timestream-query` | yes | yes | yes | 3 | +| `amp` | yes | — | yes | 2 | +| `apigatewaymanagementapi` | yes | yes | — | 2 | +| `app-mesh` | yes | — | yes | 2 | +| `budgets` | yes | — | yes | 2 | +| `cleanrooms` | yes | — | yes | 2 | +| `cloudhsm-v2` | yes | — | yes | 2 | +| `codestar-connections` | — | yes | yes | 2 | +| `connect` | yes | — | yes | 2 | +| `data-pipeline` | yes | — | yes | 2 | +| `databrew` | yes | — | yes | 2 | +| `datasync` | yes | — | yes | 2 | +| `dax` | yes | — | yes | 2 | +| `devops-agent` | yes | — | yes | 2 | +| `direct-connect` | yes | — | yes | 2 | +| `directory-service` | yes | — | yes | 2 | +| `dsql` | yes | — | yes | 2 | +| `emr-containers` | yes | — | yes | 2 | +| `fsx` | yes | — | yes | 2 | +| `greengrass` | yes | — | yes | 2 | +| `guardduty` | yes | — | yes | 2 | +| `inspector2` | yes | — | yes | 2 | +| `ivs` | yes | — | yes | 2 | +| `kinesis-analytics` | — | yes | yes | 2 | +| `kinesis-video` | yes | — | yes | 2 | +| `macie2` | yes | — | yes | 2 | +| `mediaconnect` | yes | — | yes | 2 | +| `medialive` | yes | — | yes | 2 | +| `mediapackage` | yes | — | yes | 2 | +| `mediapackagev2` | yes | — | yes | 2 | +| `mediastore-data` | yes | yes | — | 2 | +| `network-firewall` | yes | — | yes | 2 | +| `networkmanager` | yes | — | yes | 2 | +| `opensearchserverless` | yes | — | yes | 2 | +| `osis` | yes | — | yes | 2 | +| `payment-cryptography` | yes | — | yes | 2 | +| `quicksight` | yes | — | yes | 2 | +| `resiliencehub` | yes | — | yes | 2 | +| `route-53-domains` | yes | — | yes | 2 | +| `s3vectors` | yes | — | yes | 2 | +| `securityhub` | yes | — | yes | 2 | +| `service-catalog` | yes | — | yes | 2 | +| `service-catalog-appregistry` | yes | — | yes | 2 | +| `service-quotas` | yes | — | yes | 2 | +| `signer` | yes | — | yes | 2 | +| `synthetics` | yes | — | yes | 2 | +| `timestream-influxdb` | yes | — | yes | 2 | +| `vpc-lattice` | yes | — | yes | 2 | +| `workspaces` | yes | — | yes | 2 | +| `workspaces-web` | yes | — | yes | 2 | +| `accessanalyzer` | — | — | yes | 1 | +| `account-access` | — | — | yes | 1 | +| `agent-registry` | — | — | yes | 1 | +| `appconfigdata` | — | yes | — | 1 | +| `appfabric` | — | — | yes | 1 | +| `appflow` | — | — | yes | 1 | +| `appintegrations` | — | — | yes | 1 | +| `application-insights` | — | — | yes | 1 | +| `application-signals` | — | — | yes | 1 | +| `apprunner` | — | — | yes | 1 | +| `appstream` | — | — | yes | 1 | +| `arc-region-switch` | — | — | yes | 1 | +| `arc-zonal-shift` | — | — | yes | 1 | +| `auditmanager` | — | — | yes | 1 | +| `auto-scaling-plans` | — | — | yes | 1 | +| `bcm-data-exports` | — | — | yes | 1 | +| `billing` | — | — | yes | 1 | +| `chatbot` | — | — | yes | 1 | +| `chime` | — | — | yes | 1 | +| `chime-sdk-media-pipelines` | — | — | yes | 1 | +| `chime-sdk-voice` | — | — | yes | 1 | +| `cloud9` | — | — | yes | 1 | +| `clouddirectory` | yes | — | — | 1 | +| `cloudfront-keyvaluestore` | — | — | yes | 1 | +| `codecatalyst` | — | — | yes | 1 | +| `codestar-notifications` | — | — | yes | 1 | +| `compute-optimizer` | — | — | yes | 1 | +| `connectcampaigns` | yes | — | — | 1 | +| `connectcases` | — | — | yes | 1 | +| `controltower` | — | — | yes | 1 | +| `cost-and-usage-report-service` | — | — | yes | 1 | +| `cost-optimization-hub` | — | — | yes | 1 | +| `customer-profiles` | — | — | yes | 1 | +| `dataexchange` | — | — | yes | 1 | +| `datazone` | — | — | yes | 1 | +| `detective` | — | — | yes | 1 | +| `device-farm` | — | — | yes | 1 | +| `directory-service-data` | — | — | yes | 1 | +| `dlm` | — | — | yes | 1 | +| `docdb-elastic` | — | — | yes | 1 | +| `drs` | — | — | yes | 1 | +| `ec2-instance-connect` | yes | — | — | 1 | +| `ecr-public` | — | — | yes | 1 | +| `evs` | — | — | yes | 1 | +| `finspace` | — | — | yes | 1 | +| `fms` | — | — | yes | 1 | +| `gamelift` | — | — | yes | 1 | +| `global-accelerator` | — | — | yes | 1 | +| `grafana` | — | — | yes | 1 | +| `groundstation` | — | — | yes | 1 | +| `imagebuilder` | — | — | yes | 1 | +| `inspector` | — | — | yes | 1 | +| `interconnect` | — | — | yes | 1 | +| `internetmonitor` | — | — | yes | 1 | +| `invoicing` | — | — | yes | 1 | +| `ivschat` | — | — | yes | 1 | +| `kafkaconnect` | — | — | yes | 1 | +| `keyspaces` | — | — | yes | 1 | +| `kinesis-video-archived-media` | yes | — | — | 1 | +| `lambda-core` | — | — | yes | 1 | +| `lambda-microvms` | — | — | yes | 1 | +| `launch-wizard` | — | — | yes | 1 | +| `license-manager` | — | — | yes | 1 | +| `lightsail` | — | — | yes | 1 | +| `location` | — | — | yes | 1 | +| `m2` | — | — | yes | 1 | +| `mailmanager` | — | — | yes | 1 | +| `marketplace-metering` | yes | — | — | 1 | +| `mediapackage-vod` | — | — | yes | 1 | +| `mgn` | — | — | yes | 1 | +| `mpa` | — | — | yes | 1 | +| `mwaa-serverless` | — | — | yes | 1 | +| `neptune-graph` | — | — | yes | 1 | +| `networkflowmonitor` | — | — | yes | 1 | +| `networkmonitor` | — | — | yes | 1 | +| `notifications` | — | — | yes | 1 | +| `notificationscontacts` | — | — | yes | 1 | +| `oam` | — | — | yes | 1 | +| `observabilityadmin` | — | — | yes | 1 | +| `odb` | — | — | yes | 1 | +| `outposts` | — | — | yes | 1 | +| `pca-connector-ad` | — | — | yes | 1 | +| `pcs` | — | — | yes | 1 | +| `pinpoint-sms-voice-v2` | — | — | yes | 1 | +| `pricing` | — | — | yes | 1 | +| `rbin` | — | — | yes | 1 | +| `redshift-serverless` | — | — | yes | 1 | +| `resiliencehubv2` | — | — | yes | 1 | +| `resource-explorer-2` | — | — | yes | 1 | +| `rolesanywhere` | — | — | yes | 1 | +| `route53-recovery-control-config` | — | — | yes | 1 | +| `route53-recovery-readiness` | — | — | yes | 1 | +| `route53profiles` | — | — | yes | 1 | +| `rum` | — | — | yes | 1 | +| `s3files` | — | — | yes | 1 | +| `s3outposts` | — | — | yes | 1 | +| `savingsplans` | — | — | yes | 1 | +| `schemas` | — | — | yes | 1 | +| `securitylake` | — | — | yes | 1 | +| `simpledbv2` | yes | — | — | 1 | +| `ssm-contacts` | — | — | yes | 1 | +| `ssm-incidents` | — | — | yes | 1 | +| `ssm-quicksetup` | — | — | yes | 1 | +| `ssm-sap` | — | — | yes | 1 | +| `sso` | — | — | yes | 1 | +| `storage-gateway` | — | — | yes | 1 | +| `taxsettings` | — | — | yes | 1 | +| `uxc` | — | — | yes | 1 | +| `waf-regional` | — | — | yes | 1 | +| `wellarchitected` | — | — | yes | 1 | +| `workmail` | — | — | yes | 1 | +| `agent-registry-control` | — | — | — | 0 | +| `aiops` | — | — | — | 0 | +| `amplifybackend` | — | — | — | 0 | +| `amplifyuibuilder` | — | — | — | 0 | +| `application-discovery-service` | — | — | — | 0 | +| `applicationcostprofiler` | — | — | — | 0 | +| `artifact` | — | — | — | 0 | +| `b2bi` | — | — | — | 0 | +| `backup-gateway` | — | — | — | 0 | +| `backupsearch` | — | — | — | 0 | +| `bcm-dashboards` | — | — | — | 0 | +| `bcm-pricing-calculator` | — | — | — | 0 | +| `bcm-recommended-actions` | — | — | — | 0 | +| `billingconductor` | — | — | — | 0 | +| `braket` | — | — | — | 0 | +| `chime-sdk-identity` | — | — | — | 0 | +| `chime-sdk-meetings` | — | — | — | 0 | +| `chime-sdk-messaging` | — | — | — | 0 | +| `cleanroomsml` | — | — | — | 0 | +| `cloudhsm` | — | — | — | 0 | +| `cloudsearch-domain` | — | — | — | 0 | +| `cloudtrail-data` | — | — | — | 0 | +| `cloudwatch-events` | — | — | — | 0 | +| `cognito-sync` | — | — | — | 0 | +| `compute-optimizer-automation` | — | — | — | 0 | +| `connect-contact-lens` | — | — | — | 0 | +| `connectcampaignsv2` | — | — | — | 0 | +| `connecthealth` | — | — | — | 0 | +| `connectparticipant` | — | — | — | 0 | +| `controlcatalog` | — | — | — | 0 | +| `deadline` | — | — | — | 0 | +| `eks-auth` | — | — | — | 0 | +| `elementalinference` | — | — | — | 0 | +| `finspace-data` | — | — | — | 0 | +| `freetier` | — | — | — | 0 | +| `gameliftstreams` | — | — | — | 0 | +| `geo-maps` | — | — | — | 0 | +| `geo-places` | — | — | — | 0 | +| `geo-routes` | — | — | — | 0 | +| `greengrassv2` | — | — | — | 0 | +| `health` | — | — | — | 0 | +| `iam-toolbox` | — | — | — | 0 | +| `inspector-scan` | — | — | — | 0 | +| `iot-jobs-data-plane` | — | — | — | 0 | +| `iot-managed-integrations` | — | — | — | 0 | +| `iotdeviceadvisor` | — | — | — | 0 | +| `iotfleetwise` | — | — | — | 0 | +| `iotsecuretunneling` | — | — | — | 0 | +| `iotsitewise` | — | — | — | 0 | +| `iotthingsgraph` | — | — | — | 0 | +| `iottwinmaker` | — | — | — | 0 | +| `ivs-realtime` | — | — | — | 0 | +| `keyspacesstreams` | — | — | — | 0 | +| `kinesis-video-media` | — | — | — | 0 | +| `kinesis-video-signaling` | — | — | — | 0 | +| `kinesis-video-webrtc-storage` | — | — | — | 0 | +| `license-manager-linux-subscriptions` | — | — | — | 0 | +| `license-manager-user-subscriptions` | — | — | — | 0 | +| `machine-learning` | — | — | — | 0 | +| `managedblockchain-query` | — | — | — | 0 | +| `marketplace-agreement` | — | — | — | 0 | +| `marketplace-catalog` | — | — | — | 0 | +| `marketplace-commerce-analytics` | — | — | — | 0 | +| `marketplace-deployment` | — | — | — | 0 | +| `marketplace-discovery` | — | — | — | 0 | +| `marketplace-entitlement-service` | — | — | — | 0 | +| `marketplace-reporting` | — | — | — | 0 | +| `mediatailor` | — | — | — | 0 | +| `medical-imaging` | — | — | — | 0 | +| `migration-hub` | — | — | — | 0 | +| `migration-hub-refactor-spaces` | — | — | — | 0 | +| `migrationhub-config` | — | — | — | 0 | +| `migrationhuborchestrator` | — | — | — | 0 | +| `migrationhubstrategy` | — | — | — | 0 | +| `mturk` | — | — | — | 0 | +| `neptunedata` | — | — | — | 0 | +| `nova-act` | — | — | — | 0 | +| `partnercentral-account` | — | — | — | 0 | +| `partnercentral-benefits` | — | — | — | 0 | +| `partnercentral-channel` | — | — | — | 0 | +| `partnercentral-revenue-measurement` | — | — | — | 0 | +| `partnercentral-selling` | — | — | — | 0 | +| `payment-cryptography-data` | — | — | — | 0 | +| `pca-connector-scep` | — | — | — | 0 | +| `pi` | — | — | — | 0 | +| `pinpoint-email` | — | — | — | 0 | +| `pinpoint-sms-voice` | — | — | — | 0 | +| `pricing-plan-manager` | — | — | — | 0 | +| `proton` | — | — | — | 0 | +| `qconnect` | — | — | — | 0 | +| `repostspace` | — | — | — | 0 | +| `route53-recovery-cluster` | — | — | — | 0 | +| `route53globalresolver` | — | — | — | 0 | +| `rtbfabric` | — | — | — | 0 | +| `security-ir` | — | — | — | 0 | +| `securityagent` | — | — | — | 0 | +| `signer-data` | — | — | — | 0 | +| `signin` | — | — | — | 0 | +| `snow-device-management` | — | — | — | 0 | +| `snowball` | — | — | — | 0 | +| `socialmessaging` | — | — | — | 0 | +| `ssm-guiconnect` | — | — | — | 0 | +| `sso-oidc` | — | — | — | 0 | +| `supplychain` | — | — | — | 0 | +| `support-app` | — | — | — | 0 | +| `supportauthz` | — | — | — | 0 | +| `sustainability` | — | — | — | 0 | +| `tnb` | — | — | — | 0 | +| `trustedadvisor` | — | — | — | 0 | +| `wickr` | — | — | — | 0 | +| `wisdom` | — | — | — | 0 | +| `workdocs` | — | — | — | 0 | +| `workmailmessageflow` | — | — | — | 0 | +| `workspaces-instances` | — | — | — | 0 | +| `workspaces-thin-client` | — | — | — | 0 | diff --git a/docs/faq.md b/docs/faq.md index 062beb25..eb52f537 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -29,7 +29,7 @@ Developers should be able to iterate locally without cloud bills and deploy to t ### Which services are supported? -101 AWS services are scaffolded and routed; coverage depth varies. See [services-matrix.md](services-matrix.md) for the canonical list and per-service operation counts. Core services (S3, SQS, DynamoDB, Lambda, IAM, STS, SNS, CloudWatch, KMS, Secrets Manager, EventBridge, CloudFormation) pass 100% of their boto3 compatibility tests. +148 AWS services are registered and routed, 117 of them serving at least one operation; coverage depth varies. See [services-matrix.md](services-matrix.md) for the canonical list and per-service operation counts. Core services (S3, SQS, DynamoDB, Lambda, IAM, STS, SNS, CloudWatch, KMS, Secrets Manager, EventBridge, CloudFormation) pass 100% of their boto3 compatibility tests. ### Does DevCloud work on Windows? diff --git a/docs/fidelity-manifest.md b/docs/fidelity-manifest.md index 348bfd56..63fb530d 100644 --- a/docs/fidelity-manifest.md +++ b/docs/fidelity-manifest.md @@ -1,6 +1,6 @@ # Fidelity Manifest -DevCloud serves 104 AWS services, but not every operation is served the same +DevCloud registers 148 AWS services, but not every operation is served the same way. The **fidelity manifest** answers, per operation, *how much you can trust this call* — so you never have to guess whether a green response means DevCloud implemented the operation or merely made something plausible up. @@ -43,9 +43,9 @@ guaranteed across 1.x; normalizing them is a minor release. | Tier | Operations | |------|-----------:| | `hand-verified` | 4,496 | -| `auto-crud` | 948 | -| `unimplemented` | 2,031 | -| **Total** | **7,475** across 104 services | +| `auto-crud` | 1,415 | +| `unimplemented` | 3,119 | +| **Total** | **9,030** across 148 services | Examples: diff --git a/docs/services-matrix.md b/docs/services-matrix.md index 579fec74..366f40c7 100644 --- a/docs/services-matrix.md +++ b/docs/services-matrix.md @@ -1,6 +1,6 @@ # DevCloud Services Matrix -**Total**: 104 services (run `make stats`). The boto3 compatibility suite — 775 tests in `tests/compatibility/` — passes in CI. +**Total**: 148 services registered, 117 serving at least one operation (run `make stats`). The boto3 compatibility suite in `tests/compatibility/` passes in CI. A registered service that serves nothing still declines with a clean AWS error rather than letting the call reach real AWS — see [coverage.md](coverage.md). > To refresh these numbers, run `make stats` (service/handler counts) and `make test-compat` (compatibility suite), then update the line above. Note: `make stats`' "Operations" counts dispatch cases (a service carrying both Query and JSON protocols counts each), not distinct AWS operations. diff --git a/internal/admin/api.go b/internal/admin/api.go index 614fdff1..0dfff026 100644 --- a/internal/admin/api.go +++ b/internal/admin/api.go @@ -17,13 +17,18 @@ import ( type API struct { registry *plugin.Registry logCollector *LogCollector + unrouted *UnroutedCollector } -// NewAPI creates a new API. -func NewAPI(registry *plugin.Registry, logCollector *LogCollector) *API { +// NewAPI creates a new API. A nil unrouted collector is allowed; the unrouted +// route then reports an empty result rather than 404ing, so a caller reading the +// endpoint cannot mistake "not collecting" for "nothing was asked for" — the +// distinction is visible in maxServiceIds. +func NewAPI(registry *plugin.Registry, logCollector *LogCollector, unrouted *UnroutedCollector) *API { return &API{ registry: registry, logCollector: logCollector, + unrouted: unrouted, } } @@ -35,10 +40,32 @@ func (d *API) Handler() http.Handler { mux.HandleFunc("/devcloud/api/services/", d.handleServiceResources) mux.HandleFunc("/devcloud/api/logs", d.handleLogs) mux.HandleFunc("/devcloud/api/fidelity", d.handleFidelity) + mux.HandleFunc("/devcloud/api/unrouted", d.handleUnrouted) return mux } +// handleUnrouted handles GET /devcloud/api/unrouted. +// +// It answers "what did callers ask this DevCloud for that it does not +// register?" — the demand signal the coverage roadmap is gated on. +// +// It is not a complete census of unserved traffic, and docs/coverage.md says so +// where the endpoint is documented. gateway.DetectProtocol classifies anything +// it cannot identify as ("rest-xml", "s3"), and s3 is registered, so an +// unrecognisable request is routed to S3 rather than counted here. What this +// does see is the case that matters: an SDK or CLI call to a real but +// unregistered AWS service signs with that service's own name and misses the +// registry. +func (d *API) handleUnrouted(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + writeJSON(w, http.StatusOK, d.unrouted.Snapshot()) +} + // fidelityService is one service's entry in GET /devcloud/api/fidelity. type fidelityService struct { ModelBacked bool `json:"modelBacked"` diff --git a/internal/admin/api_test.go b/internal/admin/api_test.go index f8ab711c..61a4bfe1 100644 --- a/internal/admin/api_test.go +++ b/internal/admin/api_test.go @@ -74,7 +74,9 @@ func surfaceAPI(t *testing.T) http.Handler { Method: "GET", Path: "/s3/my-bucket", Status: 200, Duration: "1.000ms", Timestamp: time.Now(), Service: "s3", }) - return NewAPI(newTestRegistry(p), lc).Handler() + uc := NewUnroutedCollector(10) + uc.Add("amp") + return NewAPI(newTestRegistry(p), lc, uc).Handler() } // TestGuaranteedAdminSurface_Collections locks the wire keys of the three @@ -132,6 +134,48 @@ func TestGuaranteedAdminSurface_Fidelity(t *testing.T) { "?service= must add the per-operation tiers") } +// TestAPI_Unrouted covers the endpoint the coverage roadmap reads: a request +// for a service DevCloud does not register must be visible here, with the +// collector's limits alongside it. +func TestAPI_Unrouted(t *testing.T) { + uc := NewUnroutedCollector(10) + uc.Add("amp") + uc.Add("amp") + uc.Add("appflow") + api := NewAPI(plugin.NewRegistry(), NewLogCollector(10), uc) + + var report UnroutedReport + getJSON(t, api.Handler(), "/devcloud/api/unrouted", &report) + + require.Len(t, report.Services, 2) + assert.Equal(t, "amp", report.Services[0].ServiceID) + assert.Equal(t, 2, report.Services[0].Count) + assert.Equal(t, 10, report.MaxServiceIDs) + assert.Equal(t, 0, report.DroppedServiceIDs) +} + +// TestAPI_UnroutedWithoutCollector is the admin-enabled-but-not-collecting +// case. An empty list plus maxServiceIds 0 says "not collecting"; a 404 or a +// panic would say nothing at all. +func TestAPI_UnroutedWithoutCollector(t *testing.T) { + api := NewAPI(plugin.NewRegistry(), NewLogCollector(10), nil) + + var report UnroutedReport + getJSON(t, api.Handler(), "/devcloud/api/unrouted", &report) + + assert.Empty(t, report.Services) + assert.Equal(t, 0, report.MaxServiceIDs) +} + +func TestAPI_UnroutedRejectsNonGET(t *testing.T) { + api := NewAPI(plugin.NewRegistry(), NewLogCollector(10), NewUnroutedCollector(10)) + + w := httptest.NewRecorder() + api.Handler().ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/devcloud/api/unrouted", nil)) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) +} + // TestAPI_Services registers a mock plugin and verifies the // /devcloud/api/services endpoint returns it. func TestAPI_Services(t *testing.T) { @@ -145,7 +189,7 @@ func TestAPI_Services(t *testing.T) { } reg := newTestRegistry(p) lc := NewLogCollector(10) - api := NewAPI(reg, lc) + api := NewAPI(reg, lc, nil) req := httptest.NewRequest(http.MethodGet, "/devcloud/api/services", nil) w := httptest.NewRecorder() @@ -168,7 +212,7 @@ func TestAPI_Services(t *testing.T) { // TestAPI_Fidelity covers both shapes of the endpoint: the unfiltered summary // carries counts but no operation list, and naming a service adds its tiers. func TestAPI_Fidelity(t *testing.T) { - api := NewAPI(newTestRegistry(&mockServicePlugin{id: "s3", name: "Amazon S3"}), NewLogCollector(10)) + api := NewAPI(newTestRegistry(&mockServicePlugin{id: "s3", name: "Amazon S3"}), NewLogCollector(10), nil) req := httptest.NewRequest(http.MethodGet, "/devcloud/api/fidelity", nil) w := httptest.NewRecorder() @@ -195,7 +239,7 @@ func TestAPI_Fidelity(t *testing.T) { } func TestAPI_FidelityUnknownService(t *testing.T) { - api := NewAPI(newTestRegistry(&mockServicePlugin{id: "s3", name: "Amazon S3"}), NewLogCollector(10)) + api := NewAPI(newTestRegistry(&mockServicePlugin{id: "s3", name: "Amazon S3"}), NewLogCollector(10), nil) req := httptest.NewRequest(http.MethodGet, "/devcloud/api/fidelity?service=nosuchservice", nil) w := httptest.NewRecorder() @@ -227,7 +271,7 @@ func TestAPI_Logs(t *testing.T) { Service: "s3", }) - api := NewAPI(reg, lc) + api := NewAPI(reg, lc, nil) req := httptest.NewRequest(http.MethodGet, "/devcloud/api/logs", nil) w := httptest.NewRecorder() @@ -254,7 +298,7 @@ func TestAPI_LogsLimit(t *testing.T) { lc.Add(RequestLog{Method: "GET", Path: "/item", Status: 200, Service: "s3"}) } - api := NewAPI(reg, lc) + api := NewAPI(reg, lc, nil) req := httptest.NewRequest(http.MethodGet, "/devcloud/api/logs?limit=3", nil) w := httptest.NewRecorder() diff --git a/internal/admin/unrouted.go b/internal/admin/unrouted.go new file mode 100644 index 00000000..a02d0f59 --- /dev/null +++ b/internal/admin/unrouted.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +package admin + +import ( + "sort" + "sync" + "time" +) + +// maxUnroutedServiceIDLen bounds a recorded service ID. The ID is derived from +// caller-controlled headers, and Go's default MaxHeaderBytes is 1 MiB, so an +// untruncated key lets one client trade a header for a megabyte of retained +// memory. No real AWS service ID comes close to this. +const maxUnroutedServiceIDLen = 64 + +// UnroutedService is one service DevCloud was asked for and does not register. +type UnroutedService struct { + ServiceID string `json:"serviceId"` + Count int `json:"count"` + FirstSeen time.Time `json:"firstSeen"` + LastSeen time.Time `json:"lastSeen"` +} + +// UnroutedReport is the full answer to "what did callers ask for that we do not +// serve?". It carries the collector's limits alongside the data because an +// evidence structure that silently stopped recording is worse than no evidence. +type UnroutedReport struct { + Services []UnroutedService `json:"services"` + // MaxServiceIDs is the cap on distinct service IDs held at once. + MaxServiceIDs int `json:"maxServiceIds"` + // DroppedServiceIDs counts requests for a service ID that was not already + // tracked and arrived after the cap was reached. Non-zero means this report + // is incomplete. + DroppedServiceIDs int `json:"droppedServiceIds"` +} + +// UnroutedCollector counts requests for services the gateway does not register. +// +// Unlike LogCollector, which answers "what happened recently" from a ring +// buffer, this answers "how often, over weeks" — a ring would discard exactly +// the accumulation that makes it evidence. A map is safe here because the key +// space is bounded by what a caller can name, and the cap below bounds it again +// for callers who are not naming real services. +// +// A nil *UnroutedCollector is usable: Add is a no-op and Snapshot returns an +// empty report. That is what the gateway holds when the admin API is off — with +// nobody able to read the counts, nothing is paid to keep them. +type UnroutedCollector struct { + mu sync.RWMutex + entries map[string]*UnroutedService + maxKeys int + dropped int +} + +// NewUnroutedCollector creates a collector holding at most maxKeys distinct +// service IDs. A non-positive maxKeys would make the collector record nothing +// while reporting no drops, so it is floored at 1. +func NewUnroutedCollector(maxKeys int) *UnroutedCollector { + if maxKeys < 1 { + maxKeys = 1 + } + return &UnroutedCollector{ + entries: make(map[string]*UnroutedService), + maxKeys: maxKeys, + } +} + +// Add records one request for an unregistered service. +func (c *UnroutedCollector) Add(serviceID string) { + if c == nil || serviceID == "" { + return + } + if len(serviceID) > maxUnroutedServiceIDLen { + serviceID = serviceID[:maxUnroutedServiceIDLen] + } + + now := time.Now() + + c.mu.Lock() + defer c.mu.Unlock() + + if e, ok := c.entries[serviceID]; ok { + e.Count++ + e.LastSeen = now + return + } + if len(c.entries) >= c.maxKeys { + c.dropped++ + return + } + c.entries[serviceID] = &UnroutedService{ + ServiceID: serviceID, + Count: 1, + FirstSeen: now, + LastSeen: now, + } +} + +// Snapshot returns the current counts, most-requested first, ties broken by +// service ID so the output is stable between calls with no new traffic. +func (c *UnroutedCollector) Snapshot() UnroutedReport { + if c == nil { + return UnroutedReport{Services: []UnroutedService{}} + } + + c.mu.RLock() + defer c.mu.RUnlock() + + out := make([]UnroutedService, 0, len(c.entries)) + for _, e := range c.entries { + out = append(out, *e) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Count != out[j].Count { + return out[i].Count > out[j].Count + } + return out[i].ServiceID < out[j].ServiceID + }) + + return UnroutedReport{ + Services: out, + MaxServiceIDs: c.maxKeys, + DroppedServiceIDs: c.dropped, + } +} diff --git a/internal/admin/unrouted_test.go b/internal/admin/unrouted_test.go new file mode 100644 index 00000000..35cd05cb --- /dev/null +++ b/internal/admin/unrouted_test.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +package admin + +import ( + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestUnroutedCollector_Accumulates is the property that makes this a different +// structure from LogCollector: repeated asks add up rather than scrolling away. +func TestUnroutedCollector_Accumulates(t *testing.T) { + c := NewUnroutedCollector(100) + + for i := 0; i < 3; i++ { + c.Add("amp") + } + c.Add("appflow") + + report := c.Snapshot() + require.Len(t, report.Services, 2) + + // Most-requested first. + assert.Equal(t, "amp", report.Services[0].ServiceID) + assert.Equal(t, 3, report.Services[0].Count) + assert.Equal(t, "appflow", report.Services[1].ServiceID) + assert.Equal(t, 1, report.Services[1].Count) + + // FirstSeen is pinned at the first ask; LastSeen tracks the latest. + assert.False(t, report.Services[0].FirstSeen.IsZero()) + assert.False(t, report.Services[0].LastSeen.Before(report.Services[0].FirstSeen)) +} + +// TestUnroutedCollector_SnapshotIsStable checks the tie-break. Without it the +// map iteration order reorders equal-count services on every call, and a +// reviewer diffing two snapshots sees churn that is not traffic. +func TestUnroutedCollector_SnapshotIsStable(t *testing.T) { + c := NewUnroutedCollector(100) + for _, id := range []string{"zeta", "alpha", "mu"} { + c.Add(id) + } + + first := c.Snapshot() + for i := 0; i < 5; i++ { + assert.Equal(t, first, c.Snapshot()) + } + assert.Equal(t, []string{"alpha", "mu", "zeta"}, + []string{first.Services[0].ServiceID, first.Services[1].ServiceID, first.Services[2].ServiceID}) +} + +// TestUnroutedCollector_CapsKeysAndReportsIt covers the trust boundary: service +// IDs come from caller-controlled headers, so an unbounded map is a memory +// budget handed to whoever is calling. Dropping is acceptable; dropping +// silently is not, because the report would then read as complete. +func TestUnroutedCollector_CapsKeysAndReportsIt(t *testing.T) { + c := NewUnroutedCollector(2) + + c.Add("first") + c.Add("second") + c.Add("third") // over the cap + c.Add("first") // already tracked — still counted + + report := c.Snapshot() + require.Len(t, report.Services, 2) + assert.Equal(t, 2, report.MaxServiceIDs) + assert.Equal(t, 1, report.DroppedServiceIDs) + assert.Equal(t, "first", report.Services[0].ServiceID) + assert.Equal(t, 2, report.Services[0].Count) +} + +// TestUnroutedCollector_TruncatesLongServiceID keeps one oversized header from +// retaining an oversized map key. +func TestUnroutedCollector_TruncatesLongServiceID(t *testing.T) { + c := NewUnroutedCollector(10) + c.Add(strings.Repeat("a", 500)) + + report := c.Snapshot() + require.Len(t, report.Services, 1) + assert.Len(t, report.Services[0].ServiceID, maxUnroutedServiceIDLen) +} + +// TestUnroutedCollector_NilIsUsable is the gateway's admin-disabled case. The +// router calls Add unconditionally, so a nil collector must not panic. +func TestUnroutedCollector_NilIsUsable(t *testing.T) { + var c *UnroutedCollector + + assert.NotPanics(t, func() { c.Add("amp") }) + assert.Empty(t, c.Snapshot().Services) +} + +func TestUnroutedCollector_IgnoresEmptyServiceID(t *testing.T) { + c := NewUnroutedCollector(10) + c.Add("") + assert.Empty(t, c.Snapshot().Services) +} + +// TestUnroutedCollector_Concurrent runs under -race; the collector is written +// from the request path and read from the admin API at the same time. +func TestUnroutedCollector_Concurrent(t *testing.T) { + c := NewUnroutedCollector(100) + + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(2) + go func() { + defer wg.Done() + c.Add("amp") + }() + go func() { + defer wg.Done() + _ = c.Snapshot() + }() + } + wg.Wait() + + report := c.Snapshot() + require.Len(t, report.Services, 1) + assert.Equal(t, 50, report.Services[0].Count) +} + +// TestNewUnroutedCollector_FloorsMaxKeys guards the configuration mistake that +// would make the collector record nothing while reporting a clean report. +func TestNewUnroutedCollector_FloorsMaxKeys(t *testing.T) { + c := NewUnroutedCollector(0) + c.Add("amp") + + report := c.Snapshot() + require.Len(t, report.Services, 1) + assert.Equal(t, 1, report.MaxServiceIDs) +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 30a533b8..c694cca0 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -31,8 +31,13 @@ type Gateway struct { // and records each request after the response has been written. A nil // collector (admin API disabled) skips that wrapper entirely, keeping the // mutex lock + ring-buffer write off the request hot path. -func New(port int, registry *plugin.Registry, adminAPI http.Handler, logCollector *admin.LogCollector) *Gateway { - router := NewServiceRouter(registry) +// +// unroutedCollector gets the same nil-when-disabled treatment, but is passed +// down rather than wrapped: the count belongs on the registry-miss branch +// inside the router, which is the only place that knows a request found no +// service. +func New(port int, registry *plugin.Registry, adminAPI http.Handler, logCollector *admin.LogCollector, unroutedCollector *admin.UnroutedCollector) *Gateway { + router := NewServiceRouter(registry, unroutedCollector) // Logging middleware: records AWS API requests to logCollector. Only // installed when there is a collector to read them. diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 81de6388..2398aade 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -25,7 +25,7 @@ func TestGateway_Integration(t *testing.T) { // Build the full middleware-chained handler the same way Gateway does, // but serve it through httptest.NewServer so we exercise the real stack. - router := NewServiceRouter(reg) + router := NewServiceRouter(reg, nil) handler := ChainMiddleware(router, ErrorRecoveryMiddleware, CORSMiddleware, @@ -57,7 +57,7 @@ func TestGateway_NilLogCollector(t *testing.T) { // admin disabled → nil collector: the request must succeed without the // logging middleware (no panic, no Add). - gw := New(0, reg, http.NotFoundHandler(), nil) + gw := New(0, reg, http.NotFoundHandler(), nil, nil) rec := httptest.NewRecorder() req := httptest.NewRequest(http.MethodGet, "/", nil) @@ -70,7 +70,7 @@ func TestGateway_StartShutdown(t *testing.T) { reg := plugin.NewRegistry() lc := admin.NewLogCollector(100) adminMux := http.NewServeMux() - gw := New(0, reg, adminMux, lc) // port 0 lets the OS pick a free port + gw := New(0, reg, adminMux, lc, nil) // port 0 lets the OS pick a free port errCh := make(chan error, 1) go func() { diff --git a/internal/gateway/router.go b/internal/gateway/router.go index a50e941c..78a49f3f 100644 --- a/internal/gateway/router.go +++ b/internal/gateway/router.go @@ -12,6 +12,7 @@ import ( "net/http" "strings" + "github.com/skyoo2003/devcloud/internal/admin" "github.com/skyoo2003/devcloud/internal/plugin" "github.com/skyoo2003/devcloud/internal/shared/crud" ) @@ -20,11 +21,23 @@ import ( // ServicePlugin based on the detected AWS protocol and service ID. type ServiceRouter struct { registry *plugin.Registry + + // unrouted counts the misses below. It is nil when the admin API is off, + // and UnroutedCollector.Add tolerates that, so the miss branch stays one + // unconditional call. + // + // It undercounts by construction: DetectProtocol falls through to + // ("rest-xml", "s3") for a request it cannot classify, and s3 is + // registered, so those never reach the branch that records. A call naming a + // real but unregistered AWS service does reach it, which is the case this + // exists to measure. + unrouted *admin.UnroutedCollector } // NewServiceRouter creates a ServiceRouter backed by the given plugin Registry. -func NewServiceRouter(registry *plugin.Registry) *ServiceRouter { - return &ServiceRouter{registry: registry} +// unrouted may be nil; see the field comment. +func NewServiceRouter(registry *plugin.Registry, unrouted *admin.UnroutedCollector) *ServiceRouter { + return &ServiceRouter{registry: registry, unrouted: unrouted} } // ServeHTTP implements http.Handler. @@ -33,6 +46,7 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { p, ok := sr.registry.Get(serviceID) if !ok { + sr.unrouted.Add(serviceID) writeAWSError(w, protocol, http.StatusBadRequest, "UnknownService", "The requested service is not available: "+serviceID) return diff --git a/internal/gateway/router_test.go b/internal/gateway/router_test.go index fd28320d..022e1115 100644 --- a/internal/gateway/router_test.go +++ b/internal/gateway/router_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "testing" + "github.com/skyoo2003/devcloud/internal/admin" "github.com/skyoo2003/devcloud/internal/plugin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -49,7 +50,7 @@ func TestServiceRouter_RoutesToCorrectPlugin(t *testing.T) { ContentType: "application/xml", } reg := newRegistryWithStub("s3", resp) - router := NewServiceRouter(reg) + router := NewServiceRouter(reg, nil) req := httptest.NewRequest(http.MethodGet, "/", nil) w := httptest.NewRecorder() @@ -64,7 +65,7 @@ func TestServiceRouter_RoutesToCorrectPlugin(t *testing.T) { func TestServiceRouter_UnknownService(t *testing.T) { // Empty registry — no plugins registered. reg := plugin.NewRegistry() - router := NewServiceRouter(reg) + router := NewServiceRouter(reg, nil) // Send a JSON-protocol request targeting an unknown service. req := httptest.NewRequest(http.MethodPost, "/", nil) @@ -76,3 +77,54 @@ func TestServiceRouter_UnknownService(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Contains(t, w.Body.String(), "UnknownService") } + +// unroutedJSONRequest builds a JSON-protocol call naming service, which is how +// a boto3 client reaches a service DevCloud may or may not register. +func unroutedJSONRequest(service string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set("X-Amz-Target", service+"_20240101.DoSomething") + req.Header.Set("Content-Type", "application/x-amz-json-1.1") + return req +} + +// TestServiceRouter_RecordsUnroutedService is the demand instrument: a call to +// a service that is not registered has to land somewhere countable, or the +// coverage roadmap has no evidence to read. +func TestServiceRouter_RecordsUnroutedService(t *testing.T) { + uc := admin.NewUnroutedCollector(10) + router := NewServiceRouter(plugin.NewRegistry(), uc) + + router.ServeHTTP(httptest.NewRecorder(), unroutedJSONRequest("amp")) + router.ServeHTTP(httptest.NewRecorder(), unroutedJSONRequest("amp")) + router.ServeHTTP(httptest.NewRecorder(), unroutedJSONRequest("appflow")) + + report := uc.Snapshot() + require.Len(t, report.Services, 2) + assert.Equal(t, "amp", report.Services[0].ServiceID) + assert.Equal(t, 2, report.Services[0].Count) +} + +// TestServiceRouter_DoesNotRecordRoutedService is the other half. Counting a +// served call would make every DevCloud look like it is missing everything it +// actually has. +func TestServiceRouter_DoesNotRecordRoutedService(t *testing.T) { + uc := admin.NewUnroutedCollector(10) + reg := newRegistryWithStub("s3", &plugin.Response{StatusCode: http.StatusOK, Body: []byte("")}) + router := NewServiceRouter(reg, uc) + + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/", nil)) + + require.Equal(t, http.StatusOK, w.Code) + assert.Empty(t, uc.Snapshot().Services) +} + +// TestServiceRouter_NilUnroutedCollector is the admin-disabled configuration. +// The miss branch calls Add unconditionally, so nil must be safe there. +func TestServiceRouter_NilUnroutedCollector(t *testing.T) { + router := NewServiceRouter(plugin.NewRegistry(), nil) + + w := httptest.NewRecorder() + assert.NotPanics(t, func() { router.ServeHTTP(w, unroutedJSONRequest("amp")) }) + assert.Equal(t, http.StatusBadRequest, w.Code) +} diff --git a/scripts/demand_rank.py b/scripts/demand_rank.py new file mode 100755 index 00000000..537145e6 --- /dev/null +++ b/scripts/demand_rank.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""scripts/demand_rank.py — rank the AWS services DevCloud does not register by +how much demand three independent emulator/provider projects have already met. + +DevCloud has no usage telemetry, so it cannot measure which missing services its +own users reach for. What it can read is revealed demand: `moto`, LocalStack and +`terraform-provider-aws` each serve the populations DevCloud names as its users +(boto3 developers, CI operators, IaC engineers), and each only builds a service +when somebody asks. A service all three have built is one all three were asked +for. That is a proxy, not a measurement, and docs/demand.md says so. + +Run by hand, like scripts/download-smithy-models.sh — never in CI. The three +sources are third-party and change under us; the committed docs/demand.md is the +evidence, and this script is how it is re-derived. A source that returns nothing +exits non-zero rather than writing a file that silently under-counts, which is +the same rule download-smithy-models.sh follows. + +usage: demand_rank.py [--out docs/demand.md] [--date YYYY-MM-DD] +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import pathlib +import re +import sys +import urllib.error +import urllib.request + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent + +UPSTREAM_MODELS_API = ( + "https://api.github.com/repos/aws/aws-sdk-go-v2/contents/" + "codegen/sdk-codegen/aws-models?ref=main" +) +MOTO_COVERAGE_URL = ( + "https://raw.githubusercontent.com/getmoto/moto/master/IMPLEMENTATION_COVERAGE.md" +) +LOCALSTACK_DOCS_TREE = ( + "https://api.github.com/repos/localstack/docs/git/trees/main?recursive=1" +) +TERRAFORM_SERVICE_DIRS = ( + "https://api.github.com/repos/hashicorp/terraform-provider-aws/contents/" + "internal/service?ref=main" +) + +# Upstream model filenames whose DevCloud service ID is neither the parsed +# Smithy namespace (the model is not in tree) nor the squashed filename. Both +# are services DevCloud implements by hand under a shorter name. +FILENAME_TO_SERVICE_ID = { + "database-migration-service": "dms", + "serverlessapplicationrepository": "serverlessrepo", +} + +# Short names the sources use that no amount of separator-stripping resolves. +# Keys and values are already normalized. Two kinds live here, and both must: +# an abbreviation for a service DevCloud registers (`ce`, `logs`) would +# otherwise show as an unmatched name and make the join look broken, and an +# abbreviation for a service in the missing set (`ds`, `cur`) would otherwise +# lose a real support hit and undercount demand. +# +# Names absent from this map are absent on purpose. `elastictranscoder`, `qldb`, +# `iotanalytics` and `evidently` are services these projects support that AWS +# publishes no model for, so they cannot be in the missing set and must stay +# unmatched — silently mapping them somewhere would invent demand for a service +# that is not in the denominator. +SOURCE_NAME_ALIASES = { + # → services DevCloud already registers + "appautoscaling": "applicationautoscaling", + "ce": "costexplorer", + "cognitoidp": "cognitoidentityprovider", + "config": "configservice", + "deploy": "codedeploy", + "elasticsearch": "elasticsearchservice", + "elbv2": "elasticloadbalancingv2", + "es": "elasticsearchservice", + "events": "eventbridge", + "iotdata": "iotdataplane", + "lexmodels": "lexmodelbuildingservice", + "lexv2models": "lexmodelsv2", + "logs": "cloudwatchlogs", + "stepfunctions": "sfn", + # → services in the missing set + "cur": "costandusagereportservice", + "ds": "directoryservice", + "elb": "elasticloadbalancing", + "meteringmarketplace": "marketplacemetering", + "sdb": "simpledbv2", +} + +SOURCE_LABELS = ["moto", "LocalStack", "terraform-provider-aws"] +SOURCE_SLUGS = ["moto", "localstack", "terraform"] + + +def die(msg: str) -> None: + print(f"ERROR: {msg}", file=sys.stderr) + sys.exit(1) + + +def fetch(url: str) -> bytes: + """GET url, honouring GITHUB_TOKEN when set. + + Unauthenticated GitHub API calls are capped at 60/hour per IP, and this + script makes three of them. A rate-limited run must fail loudly: a caller + who gets an empty source and a written file has evidence that under-counts + and no way to tell. + """ + req = urllib.request.Request(url, headers={"User-Agent": "devcloud-demand-rank"}) + token = os.environ.get("GITHUB_TOKEN") + if token and "api.github.com" in url: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return resp.read() + except urllib.error.HTTPError as exc: + hint = "" + if exc.code in (403, 429): + hint = " — GitHub rate limit; set GITHUB_TOKEN and retry" + die(f"{url} returned HTTP {exc.code}{hint}") + except urllib.error.URLError as exc: + die(f"{url} unreachable: {exc.reason}") + raise AssertionError("unreachable") + + +def normalize(name: str) -> str: + """Collapse a service name to a join key. + + The four sources spell the same service several ways: AWS models and moto + use `application-autoscaling`, terraform-provider-aws uses + `applicationautoscaling`, DevCloud registers `applicationautoscaling`. + Stripping separators makes them one key. It is a lossy join, and the + unmatched counts below are what make a lossy join visible instead of + quietly deflating the support numbers. + """ + key = re.sub(r"[^a-z0-9]", "", name.lower()) + return SOURCE_NAME_ALIASES.get(key, key) + + +def registered_service_ids() -> set[str]: + """The service IDs the binary actually registers. + + Read from the Register() calls rather than from directory names, for the + same reason `make stats` does: a directory that exists but registers nothing + is not a registered service. + """ + ids: set[str] = set() + pattern = re.compile(r'DefaultRegistry\.Register\("([^"]+)"') + for path in (REPO_ROOT / "internal" / "services").rglob("*.go"): + ids.update(pattern.findall(path.read_text(encoding="utf-8"))) + if not ids: + die("found no DefaultRegistry.Register calls under internal/services") + return ids + + +def intree_model_service_ids() -> dict[str, str]: + """Map each in-tree model filename to the service ID codegen derives from it. + + codegen.detectServiceID takes the last dotted segment of the Smithy + namespace, so `com.amazonaws.acmpca#...` becomes `acmpca` — which is neither + the filename (`acm-pca`) nor anything derivable from it. + """ + out: dict[str, str] = {} + models_dir = REPO_ROOT / "smithy-models" + for path in sorted(models_dir.glob("*.json")): + model = json.loads(path.read_text(encoding="utf-8")) + for shape_id, shape in model.get("shapes", {}).items(): + if shape.get("type") == "service": + out[path.stem] = shape_id.split("#")[0].split(".")[-1] + break + if not out: + die(f"no service shapes found under {models_dir}") + return out + + +def upstream_model_names() -> list[str]: + entries = json.loads(fetch(UPSTREAM_MODELS_API)) + names = sorted(e["name"][:-5] for e in entries if e["name"].endswith(".json")) + if not names: + die("upstream model listing returned no .json files") + return names + + +def moto_services() -> set[str]: + """moto publishes one `## ` heading per implemented service. + + The file also carries `## Unimplemented:` headings, which are section + markers rather than services. Requiring a service-shaped name drops them. + """ + text = fetch(MOTO_COVERAGE_URL).decode("utf-8") + names = { + m.group(1) for m in re.finditer(r"^## ([a-z0-9][a-z0-9._-]*)\s*$", text, re.M) + } + if not names: + die("moto IMPLEMENTATION_COVERAGE.md yielded no service headings") + return names + + +def localstack_services() -> set[str]: + """LocalStack's docs carry one coverage page per service. + + The open-source repo only registers the community providers, so it + under-reports by more than half. The docs cover both tiers, which is the + list a user comparing emulators actually sees. + """ + tree = json.loads(fetch(LOCALSTACK_DOCS_TREE)) + if tree.get("truncated"): + die("LocalStack docs tree came back truncated; the service list is partial") + names = set() + pattern = re.compile(r"^content/en/references/coverage/coverage_(.+)/index\.md$") + for node in tree.get("tree", []): + m = pattern.match(node.get("path", "")) + if m: + names.add(m.group(1)) + if not names: + die("LocalStack docs tree yielded no coverage pages") + return names + + +def terraform_services() -> set[str]: + """terraform-provider-aws has one Go package per AWS service.""" + entries = json.loads(fetch(TERRAFORM_SERVICE_DIRS)) + names = {e["name"] for e in entries if e["type"] == "dir"} + if not names: + die("terraform-provider-aws internal/service listing returned no directories") + return names + + +def missing_services( + upstream: list[str], registered: set[str], intree: dict[str, str] +) -> list[str]: + """Upstream model files whose service DevCloud does not register. + + docs/coverage.md settled the denominator: one model file is one service, + because that is what an SDK client selects. + """ + missing = [] + for name in upstream: + service_id = ( + intree.get(name) or FILENAME_TO_SERVICE_ID.get(name) or normalize(name) + ) + if service_id not in registered: + missing.append(name) + return missing + + +def render( + missing: list[str], + support: dict[str, list[bool]], + counts: dict[int, int], + stats: dict[str, float], + date: str, +) -> str: + lines = [ + "# Demand for unregistered services", + "", + f"Sampled {date}. Re-derive with `python3 scripts/demand_rank.py`.", + "", + "## What this measures, and what it does not", + "", + "DevCloud has no usage telemetry, so this page does **not** measure what", + "DevCloud's users ask for. It measures *revealed* demand: which of the AWS", + "services DevCloud does not register have already been built by three", + "independent projects serving the same populations DevCloud names as its", + "users.", + "", + "| Source | What it is | Services |", + "|---|---|---|", + "| [moto](https://github.com/getmoto/moto) | Python AWS mocking library; " + "the same population as DevCloud's primary user, boto3 developers running " + f"tests | {stats['moto_total']:.0f} |", + "| [LocalStack](https://docs.localstack.cloud/references/coverage/) | Local " + "AWS emulator; community and pro tiers combined | " + f"{stats['localstack_total']:.0f} |", + "| [terraform-provider-aws](https://github.com/hashicorp/terraform-provider-aws) " + f"| One Go package per AWS service; the IaC population | {stats['terraform_total']:.0f} |", + "", + "Each of those projects adds a service because somebody asked for it. None", + "of them measured DevCloud's users. Read a high support count as *this", + "service is worth emulating to someone*, not as *our users want this*.", + "", + "The live measurement of DevCloud's own traffic is", + "`GET /devcloud/api/unrouted`, which accrues behind this page. Its ceiling", + "is documented in [coverage.md](coverage.md).", + "", + "## Readings", + "", + "| Reading | Value |", + "|---|---|", + f"| Upstream model files | {stats['upstream_total']:.0f} |", + f"| Registered by DevCloud | {stats['registered_total']:.0f} |", + f"| **R1 — missing set `M`** | **{stats['missing_total']:.0f}** |", + f"| R2 — `M` with support 3 | {counts[3]} |", + f"| R2 — `M` with support 2 | {counts[2]} |", + f"| R2 — `M` with support 1 | {counts[1]} |", + f"| R2 — `M` with support 0 | {counts[0]} |", + f"| **R2 — `M` with support ≥ 2** | **{stats['support_ge2']:.0f} " + f"({stats['support_ge2_pct']:.1f}% of `M`)** |", + "", + "Join diagnostics — a name a source publishes that matches neither `M` nor a", + "registered DevCloud service. A large number here means the name", + "normalisation is dropping matches and the support counts are too low:", + "", + "| Source | Unmatched names |", + "|---|---|", + f"| moto | {stats['moto_unmatched']:.0f} |", + f"| LocalStack | {stats['localstack_unmatched']:.0f} |", + f"| terraform-provider-aws | {stats['terraform_unmatched']:.0f} |", + "", + "## Ranking", + "", + "Ordered by support count, then name. This is the order Milestone 4 works", + "in, so stopping at any point stops on the best surface available.", + "", + "| Service | moto | LocalStack | terraform-provider-aws | Support |", + "|---|---|---|---|---|", + ] + for name in sorted(missing, key=lambda n: (-sum(support[n]), n)): + cells = ["yes" if hit else "—" for hit in support[name]] + lines.append( + f"| `{name}` | {cells[0]} | {cells[1]} | {cells[2]} | {sum(support[name])} |" + ) + lines.append("") + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", default="docs/demand.md", help="output markdown path") + parser.add_argument( + "--date", + default=datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + help="sample date recorded in the output; pass the original date to " + "reproduce a committed file byte-for-byte", + ) + args = parser.parse_args() + + registered = registered_service_ids() + intree = intree_model_service_ids() + upstream = upstream_model_names() + missing = missing_services(upstream, registered, intree) + + sources = [moto_services(), localstack_services(), terraform_services()] + + missing_keys = {normalize(n): n for n in missing} + registered_keys = {normalize(r) for r in registered} + + support = {n: [False, False, False] for n in missing} + stats: dict[str, float] = {} + for idx, (slug, names) in enumerate(zip(SOURCE_SLUGS, sources)): + unmatched = 0 + for raw in names: + key = normalize(raw) + if key in missing_keys: + support[missing_keys[key]][idx] = True + elif key not in registered_keys: + unmatched += 1 + stats[f"{slug}_total"] = len(names) + stats[f"{slug}_unmatched"] = unmatched + + counts = {n: 0 for n in range(4)} + for hits in support.values(): + counts[sum(hits)] += 1 + + stats["upstream_total"] = len(upstream) + stats["registered_total"] = len(registered) + stats["missing_total"] = len(missing) + stats["support_ge2"] = counts[2] + counts[3] + stats["support_ge2_pct"] = ( + 100.0 * stats["support_ge2"] / len(missing) if missing else 0.0 + ) + + out_path = pathlib.Path(args.out) + if not out_path.is_absolute(): + out_path = REPO_ROOT / out_path + out_path.write_text( + render(missing, support, counts, stats, args.date), encoding="utf-8" + ) + + # Print every total the decision rule reads, so an obviously-broken fetch is + # visible without opening the file. + print(f"upstream models {stats['upstream_total']:.0f}") + print(f"registered {stats['registered_total']:.0f}") + print(f"R1 missing set |M| {stats['missing_total']:.0f}") + for label, slug in zip(SOURCE_LABELS, SOURCE_SLUGS): + print( + f" {label:<24} {stats[f'{slug}_total']:>4.0f} services" + f" ({stats[f'{slug}_unmatched']:.0f} names unmatched)" + ) + for n in (3, 2, 1, 0): + print(f"R2 support {n} {counts[n]:>4}") + print( + f"R2 support >= 2 {stats['support_ge2']:.0f} " + f"({stats['support_ge2_pct']:.1f}% of M)" + ) + print(f"wrote {out_path.relative_to(REPO_ROOT)}") + + +if __name__ == "__main__": + main()