From c03ec6cf28aec9f4db7226a86380f9e56226d2bd Mon Sep 17 00:00:00 2001 From: Chris Dyson Date: Thu, 24 Sep 2026 11:48:09 +1200 Subject: [PATCH] Publish the compatibility report at /compat/ Every Overcast release attaches compat-report.json; the content sync now copies it for the last twelve releases, and the site renders an overview (headline numbers, every reason a result does not pass grouped by who has to act, a service x client matrix, changes since the previous release), a page per service with the evidence behind each failure, a static list per reason, and an explorer that filters every result by text, outcome, owner, client, tracking issue and change, with the whole view in the URL. Markdown twins, the sitemap and llms.txt cover the new pages. --- .env.example | 9 + AGENTS.md | 18 + scripts/sync-overcast-content.ts | 94 ++++ src/components/CompatCell.astro | 37 ++ src/components/CompatEmpty.astro | 15 + src/components/CompatIconSprite.astro | 34 ++ src/components/CompatOverview.astro | 406 ++++++++++++++++ src/components/CompatResultDetail.astro | 145 ++++++ src/components/SiteLayout.astro | 3 +- src/lib/compat-data.ts | 67 +++ src/lib/compat-filter.ts | 282 +++++++++++ src/lib/compat-issue-link.ts | 48 ++ src/lib/compat-report.test.ts | 313 ++++++++++++ src/lib/compat-report.ts | 601 ++++++++++++++++++++++++ src/lib/site-markdown.ts | 191 ++++++++ src/lib/site-pages.ts | 11 + src/pages/[...slug].md.ts | 35 +- src/pages/compat/[service].astro | 433 +++++++++++++++++ src/pages/compat/data/[tag].json.ts | 18 + src/pages/compat/explore.astro | 424 +++++++++++++++++ src/pages/compat/history/[tag].astro | 25 + src/pages/compat/index.astro | 16 + src/pages/compat/reason/[code].astro | 157 +++++++ src/pages/sitemap.xml.ts | 13 + src/styles/global.css | 153 ++++++ 25 files changed, 3546 insertions(+), 2 deletions(-) create mode 100644 src/components/CompatCell.astro create mode 100644 src/components/CompatEmpty.astro create mode 100644 src/components/CompatIconSprite.astro create mode 100644 src/components/CompatOverview.astro create mode 100644 src/components/CompatResultDetail.astro create mode 100644 src/lib/compat-data.ts create mode 100644 src/lib/compat-filter.ts create mode 100644 src/lib/compat-issue-link.ts create mode 100644 src/lib/compat-report.test.ts create mode 100644 src/lib/compat-report.ts create mode 100644 src/pages/compat/[service].astro create mode 100644 src/pages/compat/data/[tag].json.ts create mode 100644 src/pages/compat/explore.astro create mode 100644 src/pages/compat/history/[tag].astro create mode 100644 src/pages/compat/index.astro create mode 100644 src/pages/compat/reason/[code].astro diff --git a/.env.example b/.env.example index df13626..1632071 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,15 @@ OVERCAST_REPO=overcast-sh/overcast # OVERCAST_SOURCE_REF=v0.0.1-alpha.38 OVERCAST_EDIT_REF=main +# Compatibility report (/compat/). The sync copies each release's compat-report.json asset. +# To preview a report built from a working copy (`go run ./cmd/compat --publish-report +# compat-report.json` upstream), point at it; several paths, newest first, separated by the +# platform's path delimiter (; on Windows, : elsewhere), also preview the release-over-release +# changes. It defaults to /compat-report.json when that file exists. +# OVERCAST_COMPAT_REPORT=/path/to/compat-report.json +# Set to 1 to skip listing release reports on GitHub (offline, or local reports only). +# OVERCAST_COMPAT_OFFLINE=1 + # GitHub edit links for website-owned pages. WEBSITE_REPO=overcast-sh/website WEBSITE_EDIT_REF=main diff --git a/AGENTS.md b/AGENTS.md index 97fcb7b..fe728aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,24 @@ Gotchas: - Edit-link behavior (`EDIT_LINK_MODE`, `OVERCAST_EDIT_REF`, `WEBSITE_EDIT_REF`, etc.) is also driven by env vars in `.env.example` — see `src/lib/github-links.ts`. +### The compatibility report (/compat/) + +`syncCompat()` in the same script downloads the `compat-report.json` asset from each of the +last twelve Overcast releases that have one (GitHub API, `GITHUB_TOKEN` when set) into +`src/generated/compat/.json`, with `index.json` listing them newest first. Upstream writes +the file with `go run ./cmd/compat --publish-report`; its schema is `compat/report.schema.json` +there, and `src/lib/compat-report.ts` mirrors it. A report in another schema version is skipped. +Nothing in this step can fail the build: with no report, `/compat/` renders an empty state. + +- Pages: `/compat/` (newest release), `/compat//`, `/compat/reason//`, + `/compat/explore/` and `/compat/history//` for older releases. `/compat/data/.json` + is the explorer's compact index, built from the report at build time. +- Every number on those pages comes from the pure helpers in `src/lib/compat-report.ts`, and the + explorer's matching from `src/lib/compat-filter.ts`; both are covered by + `src/lib/compat-report.test.ts`. Change a number there, never in a page. +- Local preview: `OVERCAST_COMPAT_REPORT` (see `.env.example`) points the sync at a report built + from a working copy, and `OVERCAST_COMPAT_OFFLINE=1` skips the GitHub listing. + ## Repo layout highlights - `src/pages/` — route entry points (`index.astro`, `docs/[...slug].astro`, diff --git a/scripts/sync-overcast-content.ts b/scripts/sync-overcast-content.ts index 4685c5e..8f8f327 100644 --- a/scripts/sync-overcast-content.ts +++ b/scripts/sync-overcast-content.ts @@ -7,6 +7,7 @@ import { walk, warnMissingAllowlistedDocs, } from "../src/lib/overcast-doc-allowlist.ts"; +import { SUPPORTED_SCHEMA as COMPAT_SCHEMA } from "../src/lib/compat-report.ts"; const repo = process.env.OVERCAST_REPO || "overcast-sh/overcast"; const sourceRef = process.env.OVERCAST_SOURCE_REF || process.env.OVERCAST_TRACKING_REF || "alpha"; @@ -159,6 +160,97 @@ async function syncSupport(sourceRoot: string): Promise { return services; } +// The compatibility report (/compat/). Every Overcast release attaches a compat-report.json, +// written by `go run ./cmd/compat --publish-report` upstream; this copies one per release into +// src/generated/compat/.json and lists them, newest first, in index.json. Older releases +// stay browsable, and the newest two give the overview its release-over-release changes. +// +// Nothing here is allowed to fail the build: a network hiccup, a release without the asset, or +// a report in a schema this site does not know all degrade to fewer (or no) reports, and the +// pages render an empty state. A local file wins over the releases, so a report generated from +// a working copy can be previewed before any release carries one. +const COMPAT_ASSET = "compat-report.json"; +// Enough history to see a trend, few enough that the build does not grow with every release. +const COMPAT_RELEASES_KEPT = 12; + +type CompatIndexEntry = { tag: string; publishedAt: string | null; prerelease: boolean; source: "release" | "local" }; + +async function readCompatReport(filePath: string): Promise<{ version?: string; schema?: number } | null> { + const report = await readJsonIfExists<{ version?: string; schema?: number } | null>(filePath, null); + if (!report || report.schema !== COMPAT_SCHEMA) return null; + return report; +} + +async function fetchReleaseCompatReports(outDir: string): Promise { + const headers: Record = { Accept: "application/vnd.github+json", "User-Agent": "overcast-website-sync" }; + if (process.env.GITHUB_TOKEN) headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + const response = await fetch(`https://api.github.com/repos/${repo}/releases?per_page=50`, { headers }); + if (!response.ok) throw new Error(`GitHub releases: HTTP ${response.status}`); + const releases = (await response.json()) as Array<{ + tag_name: string; + published_at: string | null; + prerelease: boolean; + draft?: boolean; + assets: Array<{ name: string; url: string }>; + }>; + + const entries: CompatIndexEntry[] = []; + for (const release of releases) { + if (release.draft || entries.length >= COMPAT_RELEASES_KEPT) continue; + const asset = release.assets.find((candidate) => candidate.name === COMPAT_ASSET); + if (!asset) continue; + // The API URL with an octet-stream Accept, rather than browser_download_url: it takes the + // same token, so a private fork or a rate-limited runner still gets the file. + const download = await fetch(asset.url, { headers: { ...headers, Accept: "application/octet-stream" } }); + if (!download.ok) { + console.warn(`compat: ${release.tag_name}: ${COMPAT_ASSET} returned HTTP ${download.status}; skipped`); + continue; + } + const text = await download.text(); + const target = path.join(outDir, `${release.tag_name}.json`); + await fs.writeFile(target, text, "utf8"); + if (!(await readCompatReport(target))) { + console.warn(`compat: ${release.tag_name}: ${COMPAT_ASSET} is not schema ${COMPAT_SCHEMA}; skipped`); + await fs.rm(target, { force: true }); + continue; + } + entries.push({ tag: release.tag_name, publishedAt: release.published_at, prerelease: release.prerelease, source: "release" }); + } + return entries; +} + +async function syncCompat(sourceRoot: string): Promise { + const outDir = path.join(generatedDir, "compat"); + await fs.rm(outDir, { recursive: true, force: true }); + await fs.mkdir(outDir, { recursive: true }); + + // OVERCAST_COMPAT_REPORT takes one path, or several separated by the platform's path + // delimiter, newest first — two local reports are enough to preview the release-over-release + // changes before two releases carry one. + let entries: CompatIndexEntry[] = []; + const localPaths = process.env.OVERCAST_COMPAT_REPORT + ? process.env.OVERCAST_COMPAT_REPORT.split(path.delimiter).filter(Boolean) + : [path.join(sourceRoot, COMPAT_ASSET)]; + for (const localPath of localPaths) { + const local = await readCompatReport(localPath); + if (!local?.version || entries.some((entry) => entry.tag === local.version)) continue; + await fs.copyFile(localPath, path.join(outDir, `${local.version}.json`)); + entries.push({ tag: local.version, publishedAt: null, prerelease: false, source: "local" }); + console.log(`compat: using the local report at ${localPath} (${local.version})`); + } + if (process.env.OVERCAST_COMPAT_OFFLINE !== "1") { + try { + const released = await fetchReleaseCompatReports(outDir); + entries = [...entries, ...released.filter((entry) => !entries.some((e) => e.tag === entry.tag))]; + } catch (error) { + console.warn(`compat: could not list release reports (${(error as Error).message}); /compat/ shows ${entries.length ? "the local report only" : "an empty state"}`); + } + } + + await writeJson(path.join(outDir, "index.json"), entries); + return entries.length; +} + // The vendored brand assets: [path inside the branding repo, path inside this repo]. // // These stay *tracked* in git even though a script writes them. The deploy workflow never @@ -254,6 +346,7 @@ async function main() { if (brandingRoot) await syncBranding(brandingRoot); const docsCount = await countDocs(sourceRoot); const services = await syncSupport(sourceRoot); + const compatReports = await syncCompat(sourceRoot); await writeJson(path.join(generatedDir, "source-manifest.json"), { repo, @@ -263,6 +356,7 @@ async function main() { generatedAt: new Date().toISOString(), docsCount, serviceCount: services.length, + compatReports, }); } diff --git a/src/components/CompatCell.astro b/src/components/CompatCell.astro new file mode 100644 index 0000000..13f6e63 --- /dev/null +++ b/src/components/CompatCell.astro @@ -0,0 +1,37 @@ +--- +// One suite's result for one test, as a compact chip: an icon and a word, tinted by how much +// it matters. Colour is never the only signal. The word is always there, and the screen-reader +// text states the suite and the reason in full. The icon is a of CompatIconSprite, which +// every page that renders cells includes once. +import { compatIconKey, reasonShort, reasonTone, type CompatReason, type CompatResult } from "../lib/compat-report"; + +interface Props { + result: CompatResult | undefined; + reasons: CompatReason[]; + /** For the screen-reader text: "AWS SDK for Go v2". */ + suiteLabel: string; +} + +const { result, reasons, suiteLabel } = Astro.props; +const code = result?.reason; +const reason = code ? reasons.find((r) => r.code === code) : undefined; +const tone = reasonTone(code, reasons); +const spoken = !result ? `${suiteLabel}: does not run this test` : !code ? `${suiteLabel}: passes` : `${suiteLabel}: ${reason?.label ?? code}`; +--- + +{ + result ? ( + + + + {spoken} + + ) : ( + + + {spoken} + + ) +} diff --git a/src/components/CompatEmpty.astro b/src/components/CompatEmpty.astro new file mode 100644 index 0000000..749f057 --- /dev/null +++ b/src/components/CompatEmpty.astro @@ -0,0 +1,15 @@ +--- +// /compat/ before any release has published a report: say so, and point at what does exist. +import { ClipboardCheck } from "@lucide/astro"; +--- + +
+

+ compatibility report +

+

No compatibility report has been published yet.

+

+ Each release attaches one once its compatibility run has finished. Until then, the + support matrix lists which operations Overcast implements. +

+
diff --git a/src/components/CompatIconSprite.astro b/src/components/CompatIconSprite.astro new file mode 100644 index 0000000..eb4ae10 --- /dev/null +++ b/src/components/CompatIconSprite.astro @@ -0,0 +1,34 @@ +--- +// The result icons, once per page. A service page shows up to a thousand result cells, and +// inlining a Lucide SVG in each came to a third of the page's bytes; every cell now points at +// one of these symbols with . Hidden from assistive tech: each cell states its result in +// words. +import { Ban, CircleCheck, CircleDashed, CircleX, Container, CornerDownRight, FilePlus, Hourglass, Minus, Shuffle } from "@lucide/astro"; +import { COMPAT_ICON_KEYS } from "../lib/compat-report"; + +const icons: Record<(typeof COMPAT_ICON_KEYS)[number], typeof CircleCheck> = { + pass: CircleCheck, + "behaviour-mismatch": CircleX, + "not-emulated": Ban, + "quarantined-flaky": Shuffle, + "dependency-failed": CornerDownRight, + "suite-not-written": FilePlus, + candidate: Hourglass, + "sdk-lacks-api": Minus, + "needs-environment": Container, + other: CircleDashed, +}; +--- + + diff --git a/src/components/CompatOverview.astro b/src/components/CompatOverview.astro new file mode 100644 index 0000000..309cacb --- /dev/null +++ b/src/components/CompatOverview.astro @@ -0,0 +1,406 @@ +--- +// The compatibility report's overview for one release: the headline numbers, what changed since +// the release before, every reason a result did not pass (grouped by who has to act), and the +// service × suite matrix. /compat/ renders it for the newest release and /compat/history// +// for older ones; `latest` decides whether rows link to the per-service pages, which exist for +// the newest release only. +import { ArrowRight, Braces, ClipboardCheck, ListFilter, Search, TrendingDown, TrendingUp, X } from "@lucide/astro"; +import Callout from "./Callout.astro"; +import Chip from "./Chip.astro"; +import ServiceIcon from "./ServiceIcon.astro"; +import { formatDate, toIsoDate } from "../lib/format-date"; +import type { CompatRelease } from "../lib/compat-data"; +import { + WHOSE_LABELS, + WHOSE_ORDER, + WHOSE_TONE, + countByWhose, + diffReports, + isFixed, + isRegressed, + matrixRow, + measured, + operationSummaries, + percent, + rateTone, + serviceMeasured, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + shortCommit, + suiteShort, + supportServiceId, + type CompatReportIndexEntry, + type ServiceNaming, +} from "../lib/compat-report"; + +interface Props { + release: CompatRelease; + naming: ServiceNaming; + index: CompatReportIndexEntry[]; + latest: boolean; +} + +const { release, naming, index, latest } = Astro.props; +const { report, previous, entry } = release; +const reasons = reasonByCode(report); +const repoUrl = `https://github.com/${report.issueRepo}`; +const releaseUrl = entry.source === "release" ? `${repoUrl}/releases/tag/${report.version}` : null; +const jsonUrl = `/compat/data/${encodeURIComponent(report.version)}.json`; +const rawUrl = entry.source === "release" ? `${repoUrl}/releases/download/${report.version}/compat-report.json` : null; +const exploreUrl = (query = "") => `/compat/explore/${latest ? "" : `?release=${encodeURIComponent(report.version)}`}${query ? `${latest ? "?" : "&"}${query}` : ""}`; +const reasonUrl = (code: string) => (latest ? `/compat/reason/${code}/` : exploreUrl(`outcome=${code}`)); + +const headline = measured(report.totals, reasons); +const ops = operationSummaries(report); +const verified = ops.filter((op) => op.verified).length; +const whoseCounts = countByWhose(report.totals, reasons); +const blocked = report.totals.byReason["dependency-failed"] ?? 0; +const toFix = headline.ran - headline.pass; +const published = entry.publishedAt ?? report.generatedAt; + +const changes = previous ? diffReports(previous, report) : []; +const fixed = changes.filter(isFixed).length; +const regressed = changes.filter(isRegressed).length; + +const inUse = reasonsInUse(report); +const byWhose = WHOSE_ORDER.map((whose) => ({ whose, reasons: inUse.filter((reason) => reason.whose === whose) })).filter( + (group) => group.reasons.length > 0, +); + +// Services sorted by name, each with its matrix row and its headline numbers. +const services = report.services + .map((service) => { + const cells = matrixRow(service, report.suites, reasons); + const m = serviceMeasured(service, report.suites, reasons); + return { service, name: serviceDisplayName(service.id, naming), cells, pass: m.pass, ran: m.ran, gaps: m.ran - m.pass }; + }) + .sort((a, b) => a.name.localeCompare(b.name)); + +const toneVar = { success: "--oc-success", warning: "--oc-warning", danger: "--oc-danger", muted: "--oc-line", accent: "--oc-accent" } as const; +--- + +
+

+ compatibility report +

+

Tested against eight AWS clients, with the reason for every gap.

+

+ Every release runs one shared set of API tests through six AWS SDKs, the AWS CLI and the CDK against Overcast. This is + the result for {report.version}: what passes, what does not, and why. +

+ +
+
+
+
release
+
+ {releaseUrl ? {report.version} : {report.version}} + {!latest && archived} +
+
+
+
run
+
+
+ { + report.commit && ( +
+
commit
+
+ + {shortCommit(report.commit)} + +
+
+ ) + } +
+
data
+
+ + compat-report.json + +
+
+
+ { + index.length > 1 && ( + + ) + } +
+ + {/* The same dt-before-dd, visually reversed, arrangement as the support matrix's tiles. */} +
+
+
results passing, of those that measure Overcast
+
+ {percent(headline.pass, headline.ran)}%{headline.pass}/{headline.ran} +
+
+
+
operations passing in at least one client
+
{verified}/{ops.length}
+
+
+
+ results Overcast has to fix{blocked > 0 && <>, plus {blocked} tests blocked behind them} +
+
{toFix}
+
+
+
operations with no test yet
+
{whoseCounts.none}
+
+
+ +
+ + A result is one test run through one client. The pass rate counts only results that say something about Overcast: + a test not yet written for a language, an operation an SDK cannot call, or a test blocked by an earlier failure is + listed under its reason and left out of the rate. + +
+ + { + previous && ( +
+

Since {previous.version}

+
+ + + {fixed} newly passing + + + + {regressed} no longer passing + +
+
+ ) + } + +

Why results do not pass

+

+ Every result that does not pass has exactly one reason. They are grouped by who has to act on them. +

+
+ { + byWhose.map(({ whose, reasons: group }) => ( +
+
+

{WHOSE_LABELS[whose].title}

+ {whoseCounts[whose]} +
+

{WHOSE_LABELS[whose].blurb}

+ +
+ )) + } +
+ +
+
+

By service and client

+

+ Each cell is passing results over results that measure Overcast, for that service in that client. + {latest ? "Open a service for every test and the reason behind each result." : "Open the explorer for every test in this release."} +

+
+ + Search every result + +
+ +
+
+ + + + +
+ +

+ {services.length} services +

+
+ +
+ + + + + + { + report.suites.map((suite) => ( + + )) + } + + + + { + services.map(({ service, name, cells, pass, ran, gaps }) => { + const href = latest ? `/compat/${service.id}/` : exploreUrl(`q=${encodeURIComponent(service.id)}`); + return ( + + + + {cells.map((cell, i) => { + const suite = report.suites[i]; + const others = Object.entries(cell.byReason) + .map(([code, n]) => `${n} ${reasons.get(code)?.label.toLowerCase() ?? code}`) + .join(", "); + const tone = rateTone(cell.pass, cell.ran); + return ( + + ); + })} + + ); + }) + } + +
ServiceRate + + {suiteShort(suite.id)} + +
+ + + {name} + + + {percent(pass, ran)}% + + {" "} + of {ran} results passing, {gaps} Overcast {gaps === 1 ? "gap" : "gaps"} + + + {cell.cells === 0 ? ( + + + {suite.label}: no tests for this service + + ) : ( + + + + )} +
+ +
+
+ + diff --git a/src/components/CompatResultDetail.astro b/src/components/CompatResultDetail.astro new file mode 100644 index 0000000..53d6da8 --- /dev/null +++ b/src/components/CompatResultDetail.astro @@ -0,0 +1,145 @@ +--- +// Why a test did not pass in one or more clients that share the same outcome: the reason in +// words, the evidence (a parsed expected/actual diff when the failure is a generated scenario's, +// the raw messages otherwise), what it is blocked by, and where it is tracked. Callers group +// clients by reason, evidence and issue first (groupResults), so "501 in all seven SDKs" reads once. +// +// Without a tracking issue, "Report this" opens a prefilled issue carrying the marker that links +// it back to this result on the next release. It is offered for Overcast's own gaps only: a test +// not yet written for a language is tracked upstream as parity debt, and an SDK limit is nobody's bug. +import { ExternalLink } from "@lucide/astro"; +import CompatCell from "./CompatCell.astro"; +import { suiteShort, testAnchor, type CompatReason, type CompatResult } from "../lib/compat-report"; +import { newIssueUrl } from "../lib/compat-issue-link"; + +interface Props { + /** One entry per client, all with the same reason. */ + entries: { suite: { id: string; label: string }; result: CompatResult }[]; + reasons: CompatReason[]; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + version: string; + issueRepo: string; +} + +const { entries, reasons, service, serviceName, group, test, op, version, issueRepo } = Astro.props; +const first = entries[0].result; +const reason = reasons.find((r) => r.code === first.reason); +// A soaking candidate is a real assertion failure that simply does not gate yet: its evidence +// and a way to report it matter as much as for any other mismatch. +const ownedByOvercast = (reason?.whose ?? "overcast") === "overcast" || first.reason === "candidate"; +const issue = entries.find((entry) => entry.result.issue)?.result.issue; +const m = first.mismatch; +const blockedBy = [...new Set(entries.flatMap((entry) => entry.result.blockedBy ?? []))]; +// The message is evidence for a failure. For a skip it is usually just the skip sentinel +// restating the reason, so it is shown only where it says something the label does not. +const showMessages = !blockedBy.length && (ownedByOvercast || first.reason === "other-skip" || first.reason === "needs-environment"); +const messages = showMessages ? entries.filter((entry) => entry.result.error) : []; +const sameMessage = messages.length > 1 && messages.every((entry) => entry.result.error === messages[0].result.error); +const clientNames = entries.map((entry) => entry.suite.label); +const reportUrl = + issue || !ownedByOvercast + ? null + : newIssueUrl({ + issueRepo, + service, + serviceName, + group, + test, + op, + suites: entries.map((entry) => entry.suite), + reasonLabel: reason?.label ?? first.reason ?? "", + error: messages[0]?.result.error, + version, + }); +--- + +
+
+ + 1 ? clientNames.join(", ") : undefined}> + {entries.length === 1 ? entries[0].suite.label : entries.map((entry) => suiteShort(entry.suite.id)).join(", ")} + + { + issue ? ( + + #{issue.number} + {issue.title && {issue.title}} + + ) : ( + reportUrl && ( + + Report this + + ) + ) + } +
+ {reason &&

{reason.summary}

} + { + blockedBy.length > 0 && ( +

+ Blocked by{" "} + {blockedBy.map((name, i) => ( + <> + {i > 0 && ", "} + + {name} + + + ))} + . +

+ ) + } + { + m && ( +
+
check
+
+ {m.kind} + {m.path && ( + <> + {" "} + at {m.path} + + )}{" "} + on {m.op} +
+
expected
+
{m.expected}
+
actual
+
{m.actual}
+
scenario
+
+ {m.scenario} {m.step} +
+
+ ) + } + { + messages.length > 0 && ( +
+ + {messages.length === 1 || sameMessage ? "Message" : `Messages (${messages.length})`} + +
+ {(sameMessage ? messages.slice(0, 1) : messages).map((entry) => ( +
+ {!sameMessage && messages.length > 1 &&

{entry.suite.label}

} + 
+                {entry.result.error}
+
+
+ ))} +
+
+ ) + } + {first.status === "unimplemented" && messages.length === 0 &&

HTTP 501 Not Implemented

} +
diff --git a/src/components/SiteLayout.astro b/src/components/SiteLayout.astro index e5a47d5..6a821da 100644 --- a/src/components/SiteLayout.astro +++ b/src/components/SiteLayout.astro @@ -1,6 +1,6 @@ --- import "../styles/global.css"; -import { BookOpen, Download, FileText, GitCompare, Menu, Monitor, PencilLine, RotateCcwClock, Search, ShieldCheck, Star, SunMoon, X } from "@lucide/astro"; +import { BookOpen, ClipboardCheck, Download, FileText, GitCompare, Menu, Monitor, PencilLine, RotateCcwClock, Search, ShieldCheck, Star, SunMoon, X } from "@lucide/astro"; import GitHubIcon from "./GitHubIcon.astro"; import { repositoryUrl, starsUrl, websiteEditUrlFromRoute } from "../lib/github-links"; import { getSourceManifest } from "../lib/generated-content"; @@ -65,6 +65,7 @@ const primaryNav = [ { href: "/downloads/", label: "Downloads", footerLabel: "Downloads", Icon: Download }, { href: "/releases/", label: "Releases", footerLabel: "Releases", Icon: RotateCcwClock }, { href: "/support/", label: "Support Matrix", footerLabel: "Support Matrix", Icon: ShieldCheck }, + { href: "/compat/", label: "Compatibility", footerLabel: "Compatibility Report", Icon: ClipboardCheck }, { href: "/docs/migration-from-localstack/", label: "LocalStack Migration", footerLabel: "LocalStack Migration", Icon: GitCompare }, ]; // `aria-current="page"` is the only thing that tells a screen reader which nav entry is diff --git a/src/lib/compat-data.ts b/src/lib/compat-data.ts new file mode 100644 index 0000000..7c43ccc --- /dev/null +++ b/src/lib/compat-data.ts @@ -0,0 +1,67 @@ +/** + * Build-time access to the synced compatibility reports (src/generated/compat/). The + * pure helpers live in compat-report.ts; this file only reads, and caches, because every + * service page and the explorer data route read the same few megabytes. + */ +import path from "node:path"; +import fs from "node:fs/promises"; +import { getServiceSupport } from "./generated-content"; +import type { CompatReport, CompatReportIndexEntry, ServiceNaming } from "./compat-report"; + +const compatRoot = path.join(process.cwd(), "src", "generated", "compat"); + +let indexPromise: Promise | undefined; +const reportPromises = new Map>(); + +/** Every synced release, newest first. Empty when no release has published a report yet. */ +export function getCompatIndex(): Promise { + indexPromise ??= fs + .readFile(path.join(compatRoot, "index.json"), "utf8") + .then((raw) => JSON.parse(raw) as CompatReportIndexEntry[]) + .catch(() => []); + return indexPromise; +} + +export function getCompatReport(tag: string): Promise { + let promise = reportPromises.get(tag); + if (!promise) { + promise = fs + .readFile(path.join(compatRoot, `${tag}.json`), "utf8") + .then((raw) => JSON.parse(raw) as CompatReport) + .catch(() => null); + reportPromises.set(tag, promise); + } + return promise; +} + +export interface CompatRelease { + entry: CompatReportIndexEntry; + report: CompatReport; + /** The release before this one, for release-over-release changes. */ + previous: CompatReport | null; +} + +/** A release and the one before it, by tag. */ +export async function getCompatRelease(tag: string): Promise { + const index = await getCompatIndex(); + const at = index.findIndex((entry) => entry.tag === tag); + if (at < 0) return null; + const report = await getCompatReport(tag); + if (!report) return null; + const previousEntry = index[at + 1]; + return { entry: index[at], report, previous: previousEntry ? await getCompatReport(previousEntry.tag) : null }; +} + +export async function getLatestCompatRelease(): Promise { + const [latest] = await getCompatIndex(); + return latest ? getCompatRelease(latest.tag) : null; +} + +/** Display names and doc links, from the support matrix the rest of the site uses. */ +export async function getServiceNaming(): Promise { + const support = await getServiceSupport(); + return { + displayNames: new Map(support.services.map((service) => [service.service, service.displayName])), + docSlugs: new Map(support.services.map((service) => [service.service, service.docSlug ?? service.service])), + }; +} diff --git a/src/lib/compat-filter.ts b/src/lib/compat-filter.ts new file mode 100644 index 0000000..c351053 --- /dev/null +++ b/src/lib/compat-filter.ts @@ -0,0 +1,282 @@ +/** + * compat-filter — the explorer's filtering, with no DOM in it. + * + * /compat/explore/ fetches a release's ExplorerIndex and filters it as the reader types. + * Everything that decides *what matches* lives here so `npm test` can pin it: the URL + * round-trip that makes every view a link, token matching, facet counts, and ranking. + * The page script only renders what this returns. + * + * The unit of filtering is the cell (one test, one suite). A test row is shown when at + * least one of its cells passes every facet, and the facet counts are cells too, so the + * numbers on the buttons always add up to the results they would leave. + */ +import type { ExplorerIndex, ExplorerRow, Whose } from "./compat-report.ts"; + +/** An outcome is a reason code, or one of these two. */ +export const PASS = "pass"; +export const UNTESTED = "untested"; + +export type IssueFilter = "any" | "linked" | "unlinked"; +export type ChangeFilter = "any" | "fixed" | "regressed"; + +export interface FilterState { + q: string; + /** Empty means every value; otherwise the selected ones, OR-ed together. */ + outcomes: string[]; + whose: Whose[]; + suites: string[]; + issue: IssueFilter; + changed: ChangeFilter; +} + +export const EMPTY_STATE: FilterState = { q: "", outcomes: [], whose: [], suites: [], issue: "any", changed: "any" }; + +const WHOSE_VALUES: readonly Whose[] = ["overcast", "suite", "sdk", "environment", "none"]; +const ISSUE_VALUES: readonly IssueFilter[] = ["any", "linked", "unlinked"]; +const CHANGE_VALUES: readonly ChangeFilter[] = ["any", "fixed", "regressed"]; + +const list = (value: string | null) => + (value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + +/** Reads the filter from a query string. Unknown values are dropped, never thrown on. */ +export function parseState(search: string): FilterState { + const params = new URLSearchParams(search); + const issue = params.get("issue") as IssueFilter | null; + const changed = params.get("changed") as ChangeFilter | null; + return { + q: params.get("q") ?? "", + outcomes: [...new Set(list(params.get("outcome")))], + whose: [...new Set(list(params.get("whose")))].filter((w): w is Whose => WHOSE_VALUES.includes(w as Whose)), + suites: [...new Set(list(params.get("suite")))], + issue: issue && ISSUE_VALUES.includes(issue) ? issue : "any", + changed: changed && CHANGE_VALUES.includes(changed) ? changed : "any", + }; +} + +/** + * Writes the filter back as query parameters, leaving out every default so an unfiltered + * view is the bare URL. `release` is carried through untouched: it picks the data, not the + * filter. + */ +export function serializeState(state: FilterState, release?: string): string { + const params = new URLSearchParams(); + if (release) params.set("release", release); + if (state.q.trim()) params.set("q", state.q.trim()); + if (state.outcomes.length) params.set("outcome", state.outcomes.join(",")); + if (state.whose.length) params.set("whose", state.whose.join(",")); + if (state.suites.length) params.set("suite", state.suites.join(",")); + if (state.issue !== "any") params.set("issue", state.issue); + if (state.changed !== "any") params.set("changed", state.changed); + const out = params.toString(); + return out ? `?${out}` : ""; +} + +export function isFiltered(state: FilterState): boolean { + return Boolean(state.q.trim()) || state.outcomes.length > 0 || state.whose.length > 0 || state.suites.length > 0 || state.issue !== "any" || state.changed !== "any"; +} + +// ── Preparation ────────────────────────────────────────────────────────────── + +/** "CloudWatch Logs", "cloudwatch-logs" and "cloudwatchlogs" all find the same thing. */ +export const squash = (value: string): string => value.toLowerCase().replace(/[^a-z0-9#]/g, ""); + +export interface PreparedRow { + row: ExplorerRow; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + /** Everything a word may match in, lower-cased and squashed. */ + haystack: string; + /** Cell keys for the changed-since-last-release facet, one per suite (or null). */ + keys: string[]; +} + +export interface PreparedUntested { + service: string; + serviceName: string; + op: string; + code: string; + detail: string; + issue: number; + haystack: string; +} + +export interface PreparedIndex { + index: ExplorerIndex; + rows: PreparedRow[]; + untested: PreparedUntested[]; + fixed: Set; + regressed: Set; +} + +export function prepare(index: ExplorerIndex): PreparedIndex { + const rows = index.rows.map((row): PreparedRow => { + const [s, g, test, opOrEmpty, , details] = row; + const service = index.services[s]; + const group = index.groups[g]; + const op = opOrEmpty || test; + const extras = Object.values(details).flatMap(([error, issue, blocked]) => [error, issue ? `#${issue}` : "", blocked]); + const haystack = [service.id, service.name, group, test, op, ...extras].map(squash).join(" "); + const keys = index.suites.map((suite) => `${service.id}/${group}/${test}@${suite.id}`); + return { row, service: service.id, serviceName: service.name, group, test, op, haystack, keys }; + }); + const untested = index.untested.map(([s, op, code, detail, issue]): PreparedUntested => { + const service = index.services[s]; + return { + service: service.id, + serviceName: service.name, + op, + code, + detail, + issue, + haystack: [service.id, service.name, op, code, detail, issue ? `#${issue}` : ""].map(squash).join(" "), + }; + }); + return { index, rows, untested, fixed: new Set(index.changed?.fixed ?? []), regressed: new Set(index.changed?.regressed ?? []) }; +} + +// ── Evaluation ─────────────────────────────────────────────────────────────── + +export interface FacetCounts { + outcome: Record; + whose: Record; + suite: Record; + issue: Record; + changed: Record; +} + +export interface MatchedRow { + prepared: PreparedRow; + /** Per suite: does this cell pass every facet. Non-matching cells render dimmed. */ + mask: boolean[]; + rank: number; +} + +export interface Evaluation { + rows: MatchedRow[]; + untested: PreparedUntested[]; + /** Matching cells, plus matching untested operations. */ + total: number; + counts: FacetCounts; +} + +/** The words of a query, squashed. Empty words (pure punctuation) are dropped. */ +export function queryWords(q: string): string[] { + return q + .trim() + .split(/\s+/) + .map(squash) + .filter(Boolean); +} + +function textRank(words: string[], p: { op: string; test?: string; service: string }): number { + // Exact operation or test name first, then a name that starts with the first word, then + // everything else in report order. Only the first word ranks; the rest only filter. + if (!words.length) return 0; + const first = words[0]; + const names = [squash(p.op), squash(p.test ?? p.op)]; + if (names.includes(first)) return 3; + if (names.some((name) => name.startsWith(first))) return 2; + if (squash(p.service) === first) return 1; + return 0; +} + +type Cell = { outcome: string; whose: Whose; suite: string; issue: boolean; key: string }; + +/** + * Filters the prepared index. For each facet, its own counts are computed with every + * *other* facet applied, which is what makes a count say "pressing this would leave N". + */ +export function evaluate(prepared: PreparedIndex, state: FilterState): Evaluation { + const { index } = prepared; + const words = queryWords(state.q); + const outcomeOf = (cell: number) => (cell === -1 ? PASS : index.reasons[cell]?.code ?? "unknown"); + const whoseOfCell = (cell: number): Whose => (cell === -1 ? "overcast" : index.reasons[cell]?.whose ?? "overcast"); + + const outcomeOk = (c: Pick) => !state.outcomes.length || state.outcomes.includes(c.outcome); + // A pass belongs to nobody: the owner facet narrows to gaps, so a pass never survives it. + const whoseOk = (c: Pick) => !state.whose.length || (c.outcome !== PASS && state.whose.includes(c.whose)); + const suiteOk = (c: Pick) => !state.suites.length || state.suites.includes(c.suite); + const issueOk = (c: Pick) => state.issue === "any" || (state.issue === "linked") === c.issue; + const changedOk = (c: Pick) => + state.changed === "any" || (state.changed === "fixed" ? prepared.fixed.has(c.key) : prepared.regressed.has(c.key)); + const checks = { outcome: outcomeOk, whose: whoseOk, suite: suiteOk, issue: issueOk, changed: changedOk }; + type Facet = keyof typeof checks; + const passesExcept = (c: Cell, skip: Facet | null) => + (Object.keys(checks) as Facet[]).every((facet) => facet === skip || checks[facet](c as never)); + + const counts: FacetCounts = { + outcome: {}, + whose: {}, + suite: {}, + issue: { any: 0, linked: 0, unlinked: 0 }, + changed: { any: 0, fixed: 0, regressed: 0 }, + }; + const bump = (record: Record, key: string) => (record[key] = (record[key] ?? 0) + 1); + + const rows: MatchedRow[] = []; + let total = 0; + for (const p of prepared.rows) { + if (!words.every((word) => p.haystack.includes(word))) continue; + const [, , , , cells, details] = p.row; + const mask = cells.map(() => false); + cells.forEach((value, s) => { + if (value === null) return; + const cell: Cell = { + outcome: outcomeOf(value), + whose: whoseOfCell(value), + suite: index.suites[s].id, + issue: Boolean(details[s]?.[1]), + key: p.keys[s], + }; + if (passesExcept(cell, "outcome")) bump(counts.outcome, cell.outcome); + if (cell.outcome !== PASS && passesExcept(cell, "whose")) bump(counts.whose, cell.whose); + if (passesExcept(cell, "suite")) bump(counts.suite, cell.suite); + if (passesExcept(cell, "issue")) { + counts.issue.any += 1; + bump(counts.issue, cell.issue ? "linked" : "unlinked"); + } + if (passesExcept(cell, "changed")) { + counts.changed.any += 1; + if (prepared.fixed.has(cell.key)) counts.changed.fixed += 1; + if (prepared.regressed.has(cell.key)) counts.changed.regressed += 1; + } + if (passesExcept(cell, null)) { + mask[s] = true; + total += 1; + } + }); + if (mask.some(Boolean)) rows.push({ prepared: p, mask, rank: textRank(words, p) }); + } + + // Untested operations have no suite and no result. They join the outcome and owner + // facets as "untested" / "none", and drop out as soon as a suite, issue link or change + // is asked for, since none of those can describe them. + const untested: PreparedUntested[] = []; + const noSuiteFacets = !state.suites.length && state.changed === "any"; + for (const u of prepared.untested) { + if (!words.every((word) => u.haystack.includes(word))) continue; + const linked = u.issue > 0; + const cell = { outcome: UNTESTED, whose: "none" as Whose, issue: linked }; + const issueMatch = state.issue === "any" || (state.issue === "linked") === linked; + if (noSuiteFacets && issueMatch && whoseOk(cell)) bump(counts.outcome, UNTESTED); + if (noSuiteFacets && issueMatch && outcomeOk(cell)) bump(counts.whose, "none"); + if (noSuiteFacets && outcomeOk(cell) && whoseOk(cell)) { + counts.issue.any += 1; + bump(counts.issue, linked ? "linked" : "unlinked"); + } + if (noSuiteFacets && issueMatch && outcomeOk(cell) && whoseOk(cell)) { + untested.push(u); + total += 1; + } + } + + rows.sort((a, b) => b.rank - a.rank); + untested.sort((a, b) => textRank(words, { op: b.op, service: b.service }) - textRank(words, { op: a.op, service: a.service })); + return { rows, untested, total, counts }; +} diff --git a/src/lib/compat-issue-link.ts b/src/lib/compat-issue-link.ts new file mode 100644 index 0000000..82cc5be --- /dev/null +++ b/src/lib/compat-issue-link.ts @@ -0,0 +1,48 @@ +/** + * The "Report this" link on a compatibility result with no tracking issue: a new-issue form, + * prefilled, carrying the marker (``) that + * scripts/compat-issues.py in overcast-sh/overcast reads. Filing it is enough for the next + * release's report to link every listed client's result to the issue; nobody edits a mapping file. + */ + +export interface IssueLinkInput { + issueRepo: string; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + /** The clients that share this outcome. */ + suites: { id: string; label: string }[]; + reasonLabel: string; + error?: string; + version: string; +} + +/** Long enough to recognise the failure, short enough to keep the URL well under 8 KB. */ +const ERROR_LIMIT = 1500; + +/** The marker targets for one test in the given clients: service/group/test@suite. */ +export function issueMarkerTargets({ service, group, test, suites }: Pick): string[] { + return suites.map((suite) => `${service}/${group}/${test}@${suite.id}`); +} + +export function newIssueUrl(input: IssueLinkInput): string { + const { issueRepo, serviceName, op, suites, reasonLabel, error, version, group, test } = input; + const clients = suites.length === 1 ? suites[0].label : `${suites.length} clients`; + const title = `Compat: ${serviceName} ${op} in ${clients}: ${reasonLabel.toLowerCase()}`; + const message = error ? (error.length > ERROR_LIMIT ? `${error.slice(0, ERROR_LIMIT)}…` : error) : ""; + const body = [ + `**Service:** ${serviceName}`, + `**Operation:** ${op} (test \`${group}/${test}\`)`, + `**Clients:** ${suites.map((suite) => suite.label).join(", ")}`, + `**Release:** ${version}`, + `**Reported as:** ${reasonLabel}`, + "", + ...(message ? ["```", message, "```", ""] : []), + "", + ``, + ].join("\n"); + const params = new URLSearchParams({ title, body, labels: "compat" }); + return `https://github.com/${issueRepo}/issues/new?${params.toString()}`; +} diff --git a/src/lib/compat-report.test.ts b/src/lib/compat-report.test.ts new file mode 100644 index 0000000..b7522ff --- /dev/null +++ b/src/lib/compat-report.test.ts @@ -0,0 +1,313 @@ +// Unit tests for the compatibility report helpers and the explorer's filter. Run with +// `npm test`. Both modules are pure, so the fixture below is all either needs. + +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { + buildExplorerIndex, + countByWhose, + diffReports, + groupResults, + isFixed, + isRegressed, + matrixRow, + measured, + operationSummaries, + rateTone, + reasonByCode, + serviceDisplayName, + serviceDocPath, + testAnchor, + trimError, + type CompatReport, + type ServiceNaming, +} from "./compat-report.ts"; +import { EMPTY_STATE, evaluate, isFiltered, parseState, prepare, serializeState, type FilterState } from "./compat-filter.ts"; +import { issueMarkerTargets, newIssueUrl } from "./compat-issue-link.ts"; +import { compatOverviewMarkdown, compatReasonMarkdown, compatServiceMarkdown } from "./site-markdown.ts"; + +const REASONS: CompatReport["reasons"] = [ + { code: "behaviour-mismatch", label: "Behaves differently from AWS", summary: "…", whose: "overcast" }, + { code: "not-emulated", label: "Not emulated yet", summary: "…", whose: "overcast" }, + { code: "dependency-failed", label: "Blocked by an earlier step", summary: "…", whose: "overcast" }, + { code: "suite-not-written", label: "Test not written for this SDK", summary: "…", whose: "suite" }, + { code: "sdk-lacks-api", label: "Not in this SDK", summary: "…", whose: "sdk" }, + { code: "untested", label: "Not tested", summary: "…", whose: "none" }, +]; + +function report(version: string, overrides: Partial> = {}): CompatReport { + // go-sdk and rust-sdk over two S3 tests and one ELB test. `overrides` sets a cell's reason + // by "test@suite" ("" is a pass), so a second release can move individual cells. + const cell = (test: string, suite: string, fallback: string) => { + const reason = overrides[`${test}@${suite}`] ?? fallback; + return reason ? { status: "skip" as const, reason, error: `${reason} for ${test}` } : { status: "pass" as const }; + }; + return { + schema: 1, + version, + generatedAt: "2026-09-24T00:00:00Z", + issueRepo: "overcast-sh/overcast", + reasons: REASONS, + suites: [ + { id: "go-sdk", label: "AWS SDK for Go v2", kind: "sdk", language: "Go", totals: { cells: 3, pass: 2, byReason: {} } }, + { id: "rust-sdk", label: "AWS SDK for Rust", kind: "sdk", language: "Rust", totals: { cells: 3, pass: 1, byReason: {} } }, + ], + totals: { cells: 6, pass: 3, byReason: { "not-emulated": 1, "suite-not-written": 2 }, untested: 1 }, + services: [ + { + id: "s3", + totals: { cells: 4, pass: 2, byReason: {} }, + groups: [ + { + id: "s3-crud", + tests: [ + { name: "CreateBucket", op: "CreateBucket", results: { "go-sdk": cell("CreateBucket", "go-sdk", ""), "rust-sdk": cell("CreateBucket", "rust-sdk", "") } }, + { + name: "PutObjectTagged", + op: "PutObject", + results: { + "go-sdk": cell("PutObjectTagged", "go-sdk", "not-emulated"), + "rust-sdk": { ...cell("PutObjectTagged", "rust-sdk", "suite-not-written"), issue: { number: 42, url: "https://github.com/o/r/issues/42" } }, + }, + }, + ], + }, + ], + }, + { + id: "elastic-load-balancing", + totals: { cells: 2, pass: 1, byReason: {} }, + groups: [ + { + id: "elb-crud", + tests: [ + { name: "CreateLoadBalancer", op: "CreateLoadBalancer", results: { "go-sdk": cell("CreateLoadBalancer", "go-sdk", ""), "rust-sdk": cell("CreateLoadBalancer", "rust-sdk", "suite-not-written") } }, + ], + }, + ], + untested: [{ op: "DeleteLoadBalancer", group: "elb-crud", reason: "never-probe", code: "never-probe", detail: "destructive" }], + }, + ], + }; +} + +const naming: ServiceNaming = { + displayNames: new Map([ + ["s3", "S3"], + ["elbv2", "ELBv2"], + ]), + docSlugs: new Map([ + ["s3", "s3"], + ["elbv2", "elb"], + ]), +}; + +describe("compat-report", () => { + it("names services through the support matrix, aliases included", () => { + assert.equal(serviceDisplayName("s3", naming), "S3"); + assert.equal(serviceDisplayName("elastic-load-balancing", naming), "ELBv2"); + assert.equal(serviceDocPath("elastic-load-balancing", naming), "/docs/services/elb/"); + assert.equal(serviceDisplayName("servicediscovery", naming), "Cloud Map"); + assert.equal(serviceDocPath("cdk", naming), null); + assert.equal(serviceDisplayName("brand-new", naming), "brand-new"); + }); + + it("builds stable anchors", () => { + assert.equal(testAnchor("s3-crud", "PutObjectTagged"), "s3-crud--putobjecttagged"); + assert.equal(testAnchor("g", "Weird Name/1"), "g--weird-name-1"); + }); + + it("measures only the cells that say something about Overcast", () => { + const r = report("v1"); + const reasons = reasonByCode(r); + assert.deepEqual(measured(r.totals, reasons), { pass: 3, ran: 4 }); + // A blocked test never ran, so it is not charged a second time. + assert.deepEqual(measured({ ...r.totals, byReason: { ...r.totals.byReason, "dependency-failed": 5 } }, reasons), { pass: 3, ran: 4 }); + assert.deepEqual(countByWhose(r.totals, reasons), { overcast: 1, suite: 2, sdk: 0, environment: 0, none: 1 }); + }); + + it("rolls a service up per suite", () => { + const r = report("v1"); + const [go, rust] = matrixRow(r.services[0], r.suites, reasonByCode(r)); + assert.deepEqual(go, { suite: "go-sdk", pass: 1, ran: 2, cells: 2, byReason: { "not-emulated": 1 } }); + assert.deepEqual(rust, { suite: "rust-sdk", pass: 1, ran: 1, cells: 2, byReason: { "suite-not-written": 1 } }); + assert.equal(rateTone(go.pass, go.ran), "danger"); + assert.equal(rateTone(rust.pass, rust.ran), "success"); + assert.equal(rateTone(0, 0), "muted"); + }); + + it("counts an operation verified when any suite passes it", () => { + const ops = operationSummaries(report("v1")); + const put = ops.find((op) => op.op === "PutObject"); + assert.deepEqual(put, { service: "s3", op: "PutObject", verified: false, failing: true }); + assert.equal(ops.filter((op) => op.verified).length, 2); + }); + + it("diffs two releases cell by cell", () => { + const before = report("v1"); + const after = report("v2", { "PutObjectTagged@go-sdk": "", "CreateBucket@rust-sdk": "behaviour-mismatch" }); + const changes = diffReports(before, after); + assert.equal(changes.length, 2); + assert.deepEqual(changes.filter(isFixed).map((c) => `${c.test}@${c.suite}`), ["PutObjectTagged@go-sdk"]); + assert.deepEqual(changes.filter(isRegressed).map((c) => `${c.test}@${c.suite}`), ["CreateBucket@rust-sdk"]); + }); + + it("trims long errors to one line", () => { + assert.equal(trimError("a\n b"), "a b"); + assert.equal(trimError("x".repeat(10), 5), "xxxx…"); + assert.equal(trimError(undefined), ""); + }); + + it("encodes the explorer index compactly", () => { + const index = buildExplorerIndex(report("v2", { "PutObjectTagged@go-sdk": "" }), naming, report("v1")); + assert.equal(index.rows.length, 3); + const put = index.rows[1]; + assert.equal(put[2], "PutObjectTagged"); + assert.equal(put[3], "PutObject"); + assert.deepEqual(put[4], [-1, 3]); + assert.deepEqual(put[5], { 1: ["suite-not-written for PutObjectTagged", 42, ""] }); + // An op equal to its test name is not repeated. + assert.equal(index.rows[0][3], ""); + assert.deepEqual(index.untested, [[1, "DeleteLoadBalancer", "never-probe", "destructive", 0]]); + assert.equal(index.services[1].name, "ELBv2"); + assert.deepEqual(index.changed, { previous: "v1", fixed: ["s3/s3-crud/PutObjectTagged@go-sdk"], regressed: [] }); + }); + + it("keeps a reason code the report does not describe", () => { + const r = report("v1", { "CreateBucket@go-sdk": "brand-new-reason" }); + const index = buildExplorerIndex(r, naming); + const code = index.rows[0][4][0] as number; + assert.equal(index.reasons[code].code, "brand-new-reason"); + assert.equal(index.reasons.length, REASONS.length + 1); + }); +}); + +describe("compat-filter", () => { + const prepared = prepare(buildExplorerIndex(report("v2", { "PutObjectTagged@go-sdk": "" }), naming, report("v1"))); + const run = (state: Partial) => evaluate(prepared, { ...EMPTY_STATE, ...state }); + + it("round-trips the URL, dropping defaults and junk", () => { + const state: FilterState = { q: "put object", outcomes: ["not-emulated", "pass"], whose: ["overcast"], suites: ["rust-sdk"], issue: "linked", changed: "fixed" }; + const query = serializeState(state, "v2"); + assert.equal(query, "?release=v2&q=put+object&outcome=not-emulated%2Cpass&whose=overcast&suite=rust-sdk&issue=linked&changed=fixed"); + assert.deepEqual(parseState(query), state); + assert.equal(serializeState(EMPTY_STATE), ""); + assert.deepEqual(parseState("?whose=nobody&issue=maybe&changed=sideways"), EMPTY_STATE); + assert.equal(isFiltered(EMPTY_STATE), false); + assert.equal(isFiltered({ ...EMPTY_STATE, q: " " }), false); + }); + + it("shows every row and counts every cell when nothing is filtered", () => { + const result = run({}); + assert.equal(result.rows.length, 3); + assert.equal(result.untested.length, 1); + assert.equal(result.total, 7); + assert.deepEqual(result.counts.outcome, { pass: 4, "suite-not-written": 2, untested: 1 }); + assert.equal(result.counts.suite["go-sdk"], 3); + }); + + it("matches every word, across names, errors and issue numbers", () => { + assert.deepEqual(run({ q: "put object" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + assert.deepEqual(run({ q: "elbv2" }).rows.map((r) => r.prepared.test), ["CreateLoadBalancer"]); + assert.deepEqual(run({ q: "#42" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + assert.equal(run({ q: "put nothing-like-this" }).rows.length, 0); + }); + + it("ranks an exact operation name first", () => { + const rows = run({ q: "createloadbalancer" }).rows; + assert.equal(rows[0].prepared.test, "CreateLoadBalancer"); + }); + + it("combines facets, and masks the cells that do not match", () => { + const result = run({ outcomes: ["suite-not-written"], suites: ["rust-sdk"] }); + assert.deepEqual(result.rows.map((r) => r.prepared.test), ["PutObjectTagged", "CreateLoadBalancer"]); + assert.deepEqual(result.rows[0].mask, [false, true]); + assert.equal(result.untested.length, 0, "a suite filter cannot describe an untested operation"); + }); + + it("counts each facet with the others applied", () => { + const result = run({ suites: ["rust-sdk"] }); + // Outcome counts follow the suite filter... + assert.deepEqual(result.counts.outcome, { pass: 1, "suite-not-written": 2 }); + // ...and the suite counts ignore it, so the other suite's button says what it would show. + assert.equal(result.counts.suite["go-sdk"], 3); + }); + + it("leaves passes out of the owner facet", () => { + const result = run({ whose: ["suite"] }); + assert.equal(result.total, 2); + assert.equal(result.counts.outcome.pass, undefined); + }); + + it("filters on issue links and on changes since the previous release", () => { + assert.deepEqual(run({ issue: "linked" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + const fixed = run({ changed: "fixed" }); + assert.deepEqual(fixed.rows.map((r) => `${r.prepared.test}`), ["PutObjectTagged"]); + assert.deepEqual(fixed.rows[0].mask, [true, false]); + assert.equal(fixed.counts.changed.fixed, 1); + }); + + it("finds untested operations by their outcome and reason text", () => { + const result = run({ outcomes: ["untested"], q: "destructive" }); + assert.deepEqual(result.untested.map((u) => u.op), ["DeleteLoadBalancer"]); + assert.equal(result.rows.length, 0); + }); +}); + +describe("compat detail and links", () => { + it("groups clients that share an outcome, Overcast's gaps first", () => { + const r = report("v1", { "CreateBucket@go-sdk": "suite-not-written", "CreateBucket@rust-sdk": "not-emulated" }); + const test = r.services[0].groups[0].tests[0]; + const groups = groupResults(test, r.suites, reasonByCode(r)); + assert.deepEqual( + groups.map((g) => [g.reason, g.entries.map((e) => e.suite.id)]), + [ + ["not-emulated", ["rust-sdk"]], + ["suite-not-written", ["go-sdk"]], + ], + ); + const same = report("v1", { "CreateBucket@go-sdk": "not-emulated", "CreateBucket@rust-sdk": "not-emulated" }); + assert.equal(groupResults(same.services[0].groups[0].tests[0], same.suites, reasonByCode(same)).length, 1); + }); + + it("prefills an issue that links itself back through markers", () => { + const suites = [ + { id: "go-sdk", label: "AWS SDK for Go v2" }, + { id: "rust-sdk", label: "AWS SDK for Rust" }, + ]; + const input = { issueRepo: "o/r", service: "s3", serviceName: "S3", group: "s3-crud", test: "PutObjectTagged", op: "PutObject", suites, reasonLabel: "Not emulated yet", error: "x".repeat(5000), version: "v1" }; + assert.deepEqual(issueMarkerTargets(input), ["s3/s3-crud/PutObjectTagged@go-sdk", "s3/s3-crud/PutObjectTagged@rust-sdk"]); + const url = new URL(newIssueUrl(input)); + assert.equal(url.pathname, "/o/r/issues/new"); + assert.equal(url.searchParams.get("labels"), "compat"); + assert.equal(url.searchParams.get("title"), "Compat: S3 PutObject in 2 clients: not emulated yet"); + const body = url.searchParams.get("body") ?? ""; + // The marker scripts/compat-issues.py reads upstream, one target per client. + assert.match(body, //); + assert.ok(url.toString().length < 8000, "the error is trimmed so the URL stays usable"); + }); + + it("writes markdown twins from the same numbers the pages show", () => { + const r = report("v1"); + const options = { origin: "https://overcast.sh", report: r, naming, latest: true }; + const overview = compatOverviewMarkdown(options); + assert.match(overview, /^# Overcast compatibility report, v1/); + assert.match(overview, /Of the 4 results that measure Overcast, 3 pass \(75%\)/); + assert.match(overview, /\[S3\]\(https:\/\/overcast\.sh\/compat\/s3\/index\.md\) \| 66\.6% \| 1\/2 \| 1\/1 \|/); + assert.match(overview, /\[Not emulated yet\]\(https:\/\/overcast\.sh\/compat\/reason\/not-emulated\/index\.md\) \(1\)/); + + const service = compatServiceMarkdown("s3", options); + assert.match(service, /### PutObjectTagged \(PutObject\)/); + assert.match(service, /- AWS SDK for Go v2: Not emulated yet\./); + // A test not yet written is condensed to one line per client, never listed as a failure. + assert.match(service, /- Test not written for this SDK, AWS SDK for Rust: 1 test \(PutObjectTagged\)/); + assert.doesNotMatch(service, /- AWS SDK for Rust: Test not written/); + + const reason = compatReasonMarkdown("untested", options); + assert.match(reason, /- DeleteLoadBalancer: destructive/); + + // An archived release links to the explorer, since it has no per-service pages. + assert.match(compatOverviewMarkdown({ ...options, latest: false }), /\/compat\/explore\/\?release=v1&outcome=not-emulated/); + }); +}); diff --git a/src/lib/compat-report.ts b/src/lib/compat-report.ts new file mode 100644 index 0000000..3862383 --- /dev/null +++ b/src/lib/compat-report.ts @@ -0,0 +1,601 @@ +/** + * compat-report — the public compatibility report, as the site reads it. + * + * Upstream, every Overcast release ships `compat-report.json` (written by + * `go run ./cmd/compat --publish-report`, schema at compat/report.schema.json in + * overcast-sh/overcast). It is one document per release: every compat test, every + * suite's result for it, and for anything that did not pass, exactly one reason + * code that says why and whose move it is. scripts/sync-overcast-content.ts copies + * each release's report into src/generated/compat/. + * + * Everything here is pure (data in, data out) so `npm test` covers it with no build. + * The pages under src/pages/compat/ and the explorer script both work from these + * helpers, which is what keeps the static pages and the filtered view agreeing on + * what a number means. + */ + +export type Whose = "overcast" | "suite" | "sdk" | "environment" | "none"; +export type ResultStatus = "pass" | "fail" | "skip" | "unimplemented" | "na"; + +export interface CompatIssue { + number: number; + url: string; + title?: string; + state?: string; +} + +export interface CompatMismatch { + op: string; + params?: string; + kind: string; + path?: string; + expected: string; + actual: string; + scenario: string; + step: string; +} + +export interface CompatResult { + status: ResultStatus; + reason?: string; + durationMs?: number; + error?: string; + mismatch?: CompatMismatch; + blockedBy?: string[]; + issue?: CompatIssue; +} + +export interface CompatTest { + name: string; + op: string; + depends?: string[]; + requires?: string[]; + results: Record; +} + +export interface CompatGroup { + id: string; + generated?: boolean; + state?: string; + scenario?: string; + tests: CompatTest[]; +} + +export interface CompatGap { + op: string; + group: string; + reason: string; + code: string; + detail: string; + issue?: CompatIssue; +} + +export interface CompatTotals { + cells: number; + pass: number; + byReason: Record; + untested?: number; +} + +export interface CompatService { + id: string; + totals: CompatTotals; + groups: CompatGroup[]; + untested?: CompatGap[]; +} + +export interface CompatSuite { + id: string; + label: string; + kind: "sdk" | "cli" | "iac"; + language: string; + totals: CompatTotals; +} + +export interface CompatReason { + code: string; + label: string; + summary: string; + whose: Whose; +} + +export interface CompatReport { + schema: number; + version: string; + commit?: string; + generatedAt: string; + runStartedAt?: string; + runFinishedAt?: string; + issueRepo: string; + reasons: CompatReason[]; + suites: CompatSuite[]; + totals: CompatTotals; + services: CompatService[]; +} + +/** One entry of src/generated/compat/index.json, newest release first. */ +export interface CompatReportIndexEntry { + tag: string; + publishedAt: string | null; + prerelease: boolean; + /** Where the report came from: a release asset, or a local file in development. */ + source: "release" | "local"; +} + +/** The schema version this site renders. A report with any other is skipped by the sync. */ +export const SUPPORTED_SCHEMA = 1; + +// ── Owners ──────────────────────────────────────────────────────────────────── + +/** The reader-facing heading for each owner, in the order the overview lists them. */ +export const WHOSE_ORDER: readonly Whose[] = ["overcast", "suite", "sdk", "environment", "none"]; + +export const WHOSE_LABELS: Record = { + overcast: { + title: "Overcast gaps", + blurb: "Results Overcast itself has to fix: behaviour that differs from AWS, operations with no emulation yet, and the steps those block.", + }, + suite: { + title: "Test coverage gaps", + blurb: "Tests that have not been written or settled for a language yet. Overcast may well handle these calls; nothing has checked.", + }, + sdk: { + title: "SDK limits", + blurb: "Operations an SDK or the CLI has no API for, so there is nothing to call.", + }, + environment: { + title: "Environment", + blurb: "Tests that need Docker, SMTP or network access the release run does not provide.", + }, + none: { + title: "Not tested", + blurb: "Operations with no test at all. The generator declined to write one and recorded why.", + }, +}; + +export type Tone = "success" | "warning" | "danger" | "muted" | "accent"; + +/** Chip tone per owner: Overcast's gaps are the ones that bear on "will my call work". */ +export const WHOSE_TONE: Record = { + overcast: "danger", + suite: "muted", + sdk: "muted", + environment: "muted", + none: "warning", +}; + +/** Per-reason tone, finer than the owner's: a 501 is a known gap, a mismatch is a bug. */ +export function reasonTone(code: string | undefined, reasons: readonly CompatReason[]): Tone { + if (!code) return "success"; + if (code === "behaviour-mismatch") return "danger"; + if (code === "not-emulated" || code === "quarantined-flaky" || code === "dependency-failed") return "warning"; + const reason = reasons.find((r) => r.code === code); + return reason?.whose === "overcast" ? "warning" : "muted"; +} + +export function reasonByCode(report: Pick): Map { + return new Map(report.reasons.map((reason) => [reason.code, reason])); +} + +/** The owner of a reason code. Unknown codes are treated as Overcast's, the cautious reading. */ +export function whoseOf(code: string, reasons: ReadonlyMap): Whose { + return reasons.get(code)?.whose ?? "overcast"; +} + +// ── Services ────────────────────────────────────────────────────────────────── + +/** + * Compat service ids that are spelt differently from the support matrix's (and so from the + * service catalog and the docs). Everything else matches 1:1. + */ +const SUPPORT_ALIASES: Record = { + "elastic-load-balancing": "elbv2", +}; + +/** Names for compat services the support matrix has no row for. */ +const EXTRA_NAMES: Record = { + cdk: "CDK stacks", + servicediscovery: "Cloud Map", + batch: "Batch", + "elastic-load-balancing": "ELBv2", +}; + +/** The support matrix's service id for a compat service id. */ +export function supportServiceId(compatId: string): string { + return SUPPORT_ALIASES[compatId] ?? compatId; +} + +export interface ServiceNaming { + /** Display names by support-matrix service id, from service-support.json. */ + displayNames: ReadonlyMap; + /** Doc slugs by support-matrix service id. A service with no doc has no entry. */ + docSlugs: ReadonlyMap; +} + +export function serviceDisplayName(compatId: string, naming: ServiceNaming): string { + return naming.displayNames.get(supportServiceId(compatId)) ?? EXTRA_NAMES[compatId] ?? compatId; +} + +export function serviceDocPath(compatId: string, naming: ServiceNaming): string | null { + const slug = naming.docSlugs.get(supportServiceId(compatId)); + return slug ? `/docs/services/${slug}/` : null; +} + +/** A URL-safe anchor for one test row, stable across releases. */ +export function testAnchor(group: string, test: string): string { + return `${group}--${test}`.toLowerCase().replace(/[^a-z0-9-]+/g, "-"); +} + +// ── Numbers ─────────────────────────────────────────────────────────────────── + +/** + * The cells that measure Overcast: passes plus Overcast's own gaps. A test nobody has + * written, or one an SDK cannot express, says nothing about the emulator, so it stays out + * of the rate. A 501 does count against it, unlike in the suites' own pass rate: "will this + * call work against Overcast" is the question a reader brings here, and for a 501 it won't. + */ +export function measured(totals: CompatTotals, reasons: ReadonlyMap): { pass: number; ran: number } { + let gaps = 0; + for (const [code, count] of Object.entries(totals.byReason)) { + if (countsAgainst(code, reasons)) gaps += count; + } + return { pass: totals.pass, ran: totals.pass + gaps }; +} + +/** + * Reasons that are Overcast's but have no result of their own. A test blocked by an earlier + * step never ran: counting it would charge the one failure that blocked it once per dependent + * test. It is listed with Overcast's gaps, and left out of the rate and the headline count. + */ +export const UNMEASURED_REASONS: ReadonlySet = new Set(["dependency-failed"]); + +/** Whether a reason counts against Overcast's pass rate. */ +export function countsAgainst(code: string, reasons: ReadonlyMap): boolean { + return whoseOf(code, reasons) === "overcast" && !UNMEASURED_REASONS.has(code); +} + +export function percent(part: number, whole: number): number { + return whole ? Math.floor((part / whole) * 1000) / 10 : 0; +} + +/** Totals for one owner: every reason it owns, summed. */ +export function countByWhose(totals: CompatTotals, reasons: ReadonlyMap): Record { + const out: Record = { overcast: 0, suite: 0, sdk: 0, environment: 0, none: totals.untested ?? 0 }; + for (const [code, count] of Object.entries(totals.byReason)) out[whoseOf(code, reasons)] += count; + return out; +} + +export interface MatrixCell { + suite: string; + pass: number; + ran: number; + /** Every cell for this service and suite, whatever its reason. */ + cells: number; + /** Counts by reason code, for the cell's tooltip and its screen-reader text. */ + byReason: Record; +} + +/** One service's row of the service × suite matrix. */ +export function matrixRow(service: CompatService, suites: readonly CompatSuite[], reasons: ReadonlyMap): MatrixCell[] { + return suites.map((suite) => { + const cell: MatrixCell = { suite: suite.id, pass: 0, ran: 0, cells: 0, byReason: {} }; + for (const group of service.groups) { + for (const test of group.tests) { + const result = test.results[suite.id]; + if (!result) continue; + cell.cells += 1; + if (!result.reason) { + cell.pass += 1; + cell.ran += 1; + continue; + } + cell.byReason[result.reason] = (cell.byReason[result.reason] ?? 0) + 1; + if (countsAgainst(result.reason, reasons)) cell.ran += 1; + } + } + return cell; + }); +} + +/** A service's pass/ran across the given suites: the sum of its matrix row. */ +export function serviceMeasured(service: CompatService, suites: readonly CompatSuite[], reasons: ReadonlyMap): { pass: number; ran: number } { + return matrixRow(service, suites, reasons).reduce((sum, cell) => ({ pass: sum.pass + cell.pass, ran: sum.ran + cell.ran }), { pass: 0, ran: 0 }); +} + +/** Tone for a pass rate: all green, mostly there, or a real gap. No results at all is muted. */ +export function rateTone(pass: number, ran: number): Tone { + if (ran === 0) return "muted"; + if (pass === ran) return "success"; + return pass / ran >= 0.8 ? "warning" : "danger"; +} + +// ── Operations ──────────────────────────────────────────────────────────────── + +export interface OperationSummary { + service: string; + op: string; + /** At least one suite passed a test that calls it. */ + verified: boolean; + /** At least one Overcast-owned gap on a test that calls it. */ + failing: boolean; +} + +/** Per-operation rollup, for the headline "operations verified" number. */ +export function operationSummaries(report: CompatReport): OperationSummary[] { + const reasons = reasonByCode(report); + const byKey = new Map(); + for (const service of report.services) { + for (const group of service.groups) { + for (const test of group.tests) { + const key = `${service.id}/${test.op}`; + const summary = byKey.get(key) ?? { service: service.id, op: test.op, verified: false, failing: false }; + for (const result of Object.values(test.results)) { + if (!result.reason) summary.verified = true; + else if (countsAgainst(result.reason, reasons)) summary.failing = true; + } + byKey.set(key, summary); + } + } + } + return [...byKey.values()]; +} + +// ── Release-over-release ───────────────────────────────────────────────────── + +export interface CellChange { + service: string; + group: string; + test: string; + op: string; + suite: string; + /** Reason code before, "" for a pass, null when the cell did not exist. */ + from: string | null; + to: string | null; +} + +function cellMap(report: CompatReport): Map { + const out = new Map(); + for (const service of report.services) { + for (const group of service.groups) { + for (const test of group.tests) { + for (const [suite, result] of Object.entries(test.results)) { + out.set(`${service.id}/${group.id}/${test.name}@${suite}`, { service: service.id, group: group.id, test, suite, reason: result.reason ?? "" }); + } + } + } + } + return out; +} + +/** + * Every cell whose outcome moved between two releases: newly passing, newly failing, or + * changed from one reason to another. Cells that appear or disappear (a test added or + * retired) are included with a null side, so a caller can decide whether they count. + */ +export function diffReports(previous: CompatReport, current: CompatReport): CellChange[] { + const before = cellMap(previous); + const after = cellMap(current); + const changes: CellChange[] = []; + for (const [key, cell] of after) { + const old = before.get(key); + const from = old ? old.reason : null; + if (from === cell.reason) continue; + changes.push({ service: cell.service, group: cell.group, test: cell.test.name, op: cell.test.op, suite: cell.suite, from, to: cell.reason }); + } + for (const [key, cell] of before) { + if (after.has(key)) continue; + changes.push({ service: cell.service, group: cell.group, test: cell.test.name, op: cell.test.op, suite: cell.suite, from: cell.reason, to: null }); + } + return changes; +} + +/** A change that made a previously-running cell pass. */ +export const isFixed = (change: CellChange): boolean => change.to === "" && change.from !== null && change.from !== ""; +/** A change that turned a passing cell into anything else. */ +export const isRegressed = (change: CellChange): boolean => change.from === "" && change.to !== null && change.to !== ""; + +// ── The explorer's index ───────────────────────────────────────────────────── + +/** + * The compact form the explorer fetches: one row per test, one slot per suite. Strings that + * repeat (services, groups, reasons, suites) are dictionary-encoded, so ~1,200 tests across + * eight suites come to a small fraction of the full report. + * + * A cell is a reason index, -1 for a pass, or null when the suite does not run the test. + * Errors are trimmed: the explorer shows enough to recognise the failure and links to the + * service page for the whole message. + */ +export interface ExplorerIndex { + version: string; + /** owner/repo that issue numbers refer to. */ + issueRepo: string; + suites: { id: string; label: string }[]; + reasons: CompatReason[]; + services: { id: string; name: string }[]; + groups: string[]; + /** [serviceIdx, groupIdx, test, op ("" when equal to test), cells, details] */ + rows: ExplorerRow[]; + /** [serviceIdx, op, gap code, detail, issue number or 0] */ + untested: [number, string, string, string, number][]; + /** "service/group/test@suite" of every cell whose outcome moved since the previous release. */ + changed?: { fixed: string[]; regressed: string[]; previous: string }; +} + +export type ExplorerCell = number | null; +/** Per-suite detail, keyed by suite index: [error snippet, issue number or 0, blocked by]. */ +export type ExplorerDetail = Record; +export type ExplorerRow = [number, number, string, string, ExplorerCell[], ExplorerDetail]; + +export const EXPLORER_ERROR_LIMIT = 240; + +export function trimError(error: string | undefined, limit = EXPLORER_ERROR_LIMIT): string { + if (!error) return ""; + const flat = error.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit - 1)}…` : flat; +} + +export function buildExplorerIndex(report: CompatReport, naming: ServiceNaming, previous?: CompatReport): ExplorerIndex { + const reasonIndex = new Map(report.reasons.map((reason, i) => [reason.code, i])); + const suiteIndex = new Map(report.suites.map((suite, i) => [suite.id, i])); + const groups: string[] = []; + const groupIndex = new Map(); + const rows: ExplorerRow[] = []; + const untested: ExplorerIndex["untested"] = []; + + report.services.forEach((service, serviceIdx) => { + for (const group of service.groups) { + let g = groupIndex.get(group.id); + if (g === undefined) { + g = groups.push(group.id) - 1; + groupIndex.set(group.id, g); + } + for (const test of group.tests) { + const cells: ExplorerCell[] = report.suites.map(() => null); + const details: ExplorerDetail = {}; + for (const [suiteId, result] of Object.entries(test.results)) { + const s = suiteIndex.get(suiteId); + if (s === undefined) continue; + if (!result.reason) { + cells[s] = -1; + continue; + } + // An unknown code would index nothing; it is appended so it still renders. + let r = reasonIndex.get(result.reason); + if (r === undefined) { + r = reasonIndex.size; + reasonIndex.set(result.reason, r); + } + cells[s] = r; + const error = trimError(result.error); + const issue = result.issue?.number ?? 0; + const blocked = result.blockedBy?.join(", ") ?? ""; + if (error || issue || blocked) details[s] = [error, issue, blocked]; + } + rows.push([serviceIdx, g, test.name, test.op === test.name ? "" : test.op, cells, details]); + } + } + for (const gap of service.untested ?? []) { + untested.push([serviceIdx, gap.op, gap.code, gap.detail, gap.issue?.number ?? 0]); + } + }); + + const reasons = [...report.reasons]; + for (const [code, i] of reasonIndex) { + if (i >= report.reasons.length) reasons[i] = { code, label: code, summary: "", whose: "overcast" }; + } + + const index: ExplorerIndex = { + version: report.version, + issueRepo: report.issueRepo, + suites: report.suites.map((suite) => ({ id: suite.id, label: suite.label })), + reasons, + services: report.services.map((service) => ({ id: service.id, name: serviceDisplayName(service.id, naming) })), + groups, + rows, + untested, + }; + if (previous) { + const changes = diffReports(previous, report); + const key = (c: CellChange) => `${c.service}/${c.group}/${c.test}@${c.suite}`; + index.changed = { + previous: previous.version, + fixed: changes.filter(isFixed).map(key), + regressed: changes.filter(isRegressed).map(key), + }; + } + return index; +} + +// ── Presentation ───────────────────────────────────────────────────────────── + +/** Column headings for the suites: a matrix of eight full SDK names does not fit a phone. */ +export const SUITE_SHORT: Record = { + "node-js-sdk": "JS", + "python-sdk": "Python", + "go-sdk": "Go", + cli: "CLI", + cdk: "CDK", + "java-sdk": "Java", + "dotnet-sdk": ".NET", + "rust-sdk": "Rust", +}; + +export const suiteShort = (id: string): string => SUITE_SHORT[id] ?? id; + +/** The word in a matrix cell. The full label and summary come from the report itself. */ +export const REASON_SHORT: Record = { + "behaviour-mismatch": "Differs", + "not-emulated": "501", + "quarantined-flaky": "Flaky", + "dependency-failed": "Blocked", + "suite-not-written": "No test", + candidate: "Soaking", + "not-reported": "Missing", + "sdk-lacks-api": "N/A", + "needs-environment": "Env", + "other-skip": "Skipped", + untested: "Untested", +}; + +export const reasonShort = (code: string | undefined): string => (code ? (REASON_SHORT[code] ?? code) : "Pass"); + +/** Symbols in CompatIconSprite, one per reason that has its own glyph. */ +export const COMPAT_ICON_KEYS = [ + "pass", + "behaviour-mismatch", + "not-emulated", + "quarantined-flaky", + "dependency-failed", + "suite-not-written", + "candidate", + "sdk-lacks-api", + "needs-environment", + "other", +] as const; + +/** The sprite symbol for a result: its reason's own glyph, or a neutral one. */ +export function compatIconKey(code: string | undefined): (typeof COMPAT_ICON_KEYS)[number] { + if (!code) return "pass"; + return (COMPAT_ICON_KEYS as readonly string[]).includes(code) ? (code as (typeof COMPAT_ICON_KEYS)[number]) : "other"; +} + +/** Every reason that has at least one cell, in the report's own order. */ +export function reasonsInUse(report: CompatReport): CompatReason[] { + return report.reasons.filter((reason) => + reason.code === "untested" ? (report.totals.untested ?? 0) > 0 : (report.totals.byReason[reason.code] ?? 0) > 0, + ); +} + +/** The count for one reason, untested included. */ +export function reasonCount(report: CompatReport, code: string): number { + return code === "untested" ? (report.totals.untested ?? 0) : (report.totals.byReason[code] ?? 0); +} + +/** A short commit for display. */ +export const shortCommit = (sha: string | undefined): string => (sha ? sha.slice(0, 9) : ""); + +export interface ResultGroup { + reason: string; + entries: { suite: { id: string; label: string }; result: CompatResult }[]; +} + +/** + * A test's non-passing results, grouped so that clients sharing an outcome read once: same + * reason, same parsed evidence, same tracking issue. Messages may still differ (every SDK words + * a 501 its own way); the detail view lists them together. Groups keep suite order, and + * Overcast's own gaps come first. + */ +export function groupResults(test: CompatTest, suites: readonly { id: string; label: string }[], reasons: ReadonlyMap): ResultGroup[] { + const groups = new Map(); + for (const suite of suites) { + const result = test.results[suite.id]; + if (!result?.reason) continue; + const m = result.mismatch; + const key = [result.reason, result.issue?.number ?? 0, m ? `${m.kind}|${m.path}|${m.expected}|${m.actual}` : "", (result.blockedBy ?? []).join(",")].join("|"); + const group = groups.get(key) ?? { reason: result.reason, entries: [] }; + group.entries.push({ suite: { id: suite.id, label: suite.label }, result }); + groups.set(key, group); + } + const rank = (code: string) => (whoseOf(code, reasons) === "overcast" ? 0 : 1); + return [...groups.values()].sort((a, b) => rank(a.reason) - rank(b.reason)); +} diff --git a/src/lib/site-markdown.ts b/src/lib/site-markdown.ts index 915c6cb..3dc1b77 100644 --- a/src/lib/site-markdown.ts +++ b/src/lib/site-markdown.ts @@ -18,6 +18,22 @@ * build and no collection. */ import { coverageTierLabel, type ServiceSupportManifest } from "./generated-content.ts"; +import { + WHOSE_LABELS, + WHOSE_ORDER, + countByWhose, + matrixRow, + measured, + percent, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + serviceMeasured, + type CompatReason, + type CompatReport, + type ServiceNaming, +} from "./compat-report.ts"; export interface MarkdownPageOptions { /** Absolute origin, so a reader that fetched this file can follow every link in it. */ @@ -197,3 +213,178 @@ export function docsIndexMarkdown({ sections, origin }: DocsIndexOptions): strin ...blocks, ]); } + +// ── Compatibility report ───────────────────────────────────────────────────── + +export interface CompatMarkdownOptions extends MarkdownPageOptions { + report: CompatReport; + naming: ServiceNaming; + /** The newest release has per-service pages; an archived one links to the explorer. */ + latest: boolean; +} + +/** "1 result is" / "3 results are". */ +function plural(n: number, one: string, many: string): string { + return `${n} ${n === 1 ? one : many}`; +} + +/** One reason as a markdown line, for the "why" sections. */ +function reasonLine(report: CompatReport, reason: CompatReason, href: string): string { + return `- [${reason.label}](${href}) (${reasonCount(report, reason.code)}): ${reason.summary}`; +} + +/** + * /compat/index.md (and /compat/history//index.md) — the overview: the headline numbers, + * every reason by owner, and the service × client matrix as a table. + */ +export function compatOverviewMarkdown({ report, naming, latest, origin }: CompatMarkdownOptions): string { + const reasons = reasonByCode(report); + const headline = measured(report.totals, reasons); + const whose = countByWhose(report.totals, reasons); + const inUse = reasonsInUse(report); + const reasonHref = (code: string) => + latest + ? absolute(origin, `/compat/reason/${code}/index.md`) + : absolute(origin, `/compat/explore/?release=${encodeURIComponent(report.version)}&outcome=${code}`); + + const whyBlocks = WHOSE_ORDER.map((owner) => { + const group = inUse.filter((reason) => reason.whose === owner); + if (!group.length) return ""; + return [ + `### ${WHOSE_LABELS[owner].title} (${whose[owner]})`, + "", + WHOSE_LABELS[owner].blurb, + "", + ...group.map((reason) => reasonLine(report, reason, reasonHref(reason.code))), + ].join("\n"); + }).filter(Boolean); + + const services = [...report.services].sort((a, b) => + serviceDisplayName(a.id, naming).localeCompare(serviceDisplayName(b.id, naming)), + ); + const rows = services.map((service) => { + const m = serviceMeasured(service, report.suites, reasons); + const name = serviceDisplayName(service.id, naming); + const cells = matrixRow(service, report.suites, reasons).map((cell) => + cell.cells === 0 ? "—" : cell.ran ? `${cell.pass}/${cell.ran}` : "no test", + ); + const label = latest ? `[${name}](${absolute(origin, `/compat/${service.id}/index.md`)})` : name; + return [label, `${percent(m.pass, m.ran)}%`, ...cells]; + }); + const releaseUrl = `https://github.com/${report.issueRepo}/releases/tag/${report.version}`; + + return page( + `Overcast compatibility report, ${report.version}`, + "> Tested compatibility across six AWS SDKs, the AWS CLI and the CDK, with the reason for every result that does not pass.", + [ + `Of the ${headline.ran} results that measure Overcast, ${headline.pass} pass (${percent(headline.pass, headline.ran)}%). ${plural(headline.ran - headline.pass, "result is", "results are")} for Overcast to fix, ${plural(whose.suite, "is a test", "are tests")} not yet written or settled for a client, and ${plural(whose.none, "operation has", "operations have")} no test. A test not yet written for a language, an operation an SDK cannot call, or a test blocked by an earlier failure is left out of the rate.`, + `The full data is the \`compat-report.json\` asset on the release, ${releaseUrl}. Every result can be searched at ${absolute(origin, "/compat/explore/")}.`, + "## Why results do not pass", + ...whyBlocks, + "## By service and client", + "Passing results over results that measure Overcast.", + table(["Service", "Rate", ...report.suites.map((suite) => suite.label)], rows, "_No results in this report._"), + ], + ); +} + +/** + * /compat//index.md — every test Overcast does not yet pass, with the reason and the + * evidence, per client. Results that say nothing about Overcast (a test not yet written for a + * language, an SDK with no API) are condensed to one line per reason and client, or a service + * with a young Rust suite would bury its three real failures under sixty identical lines. + */ +export function compatServiceMarkdown(serviceId: string, { report, naming, origin }: CompatMarkdownOptions): string { + const service = report.services.find((s) => s.id === serviceId); + const name = serviceDisplayName(serviceId, naming); + if (!service) return page(`${name} compatibility`, "> No results for this service in this report.", []); + const reasons = reasonByCode(report); + const m = serviceMeasured(service, report.suites, reasons); + const labels = new Map(report.suites.map((suite) => [suite.id, suite.label])); + const tests = service.groups.flatMap((group) => group.tests.map((test) => ({ group: group.id, test }))); + const detailed = (code: string) => reasons.get(code)?.whose === "overcast" || code === "candidate"; + + const blocks = tests + .filter(({ test }) => Object.values(test.results).some((result) => result.reason && detailed(result.reason))) + .map(({ group, test }) => { + const lines = Object.entries(test.results) + .filter(([, result]) => result.reason && detailed(result.reason)) + .map(([suite, result]) => { + const reason = reasons.get(result.reason ?? ""); + const issue = result.issue ? ` Tracked in ${result.issue.url}.` : ""; + let evidence = ""; + if (result.mismatch) { + const at = result.mismatch.path ? ` at \`${result.mismatch.path}\`` : ""; + evidence = ` ${result.mismatch.kind}${at}: expected ${result.mismatch.expected}, actual ${result.mismatch.actual}.`; + } else if (result.blockedBy?.length) { + evidence = ` Blocked by ${result.blockedBy.join(", ")}.`; + } + return `- ${labels.get(suite) ?? suite}: ${reason?.label ?? result.reason}.${evidence}${issue}`; + }); + const heading = `### ${test.name}${test.op !== test.name ? ` (${test.op})` : ""}`; + return [heading, "", `Group \`${group}\`.`, "", ...lines].join("\n"); + }); + + // Everything else, one line per reason and client: "AWS SDK for Rust: 60 tests (CreateUser, …)". + const condensed = new Map(); + for (const { test } of tests) { + for (const [suite, result] of Object.entries(test.results)) { + if (!result.reason || detailed(result.reason)) continue; + const key = `${reasons.get(result.reason)?.label ?? result.reason}|${labels.get(suite) ?? suite}`; + condensed.set(key, [...(condensed.get(key) ?? []), test.name]); + } + } + const otherLines = [...condensed].map(([key, names]) => { + const [label, client] = key.split("|"); + return `- ${label}, ${client}: ${names.length} ${names.length === 1 ? "test" : "tests"} (${names.join(", ")})`; + }); + + const untested = (service.untested ?? []).map((gap) => `- ${gap.op} (\`${gap.code}\`): ${gap.detail}`); + + return page(`${name} compatibility, ${report.version}`, `> Compatibility test results for ${name} in Overcast ${report.version}.`, [ + `${tests.length} tests. Of the ${m.ran} results that measure Overcast, ${m.pass} pass (${percent(m.pass, m.ran)}%). ${blocks.length} tests have a result Overcast has to act on, listed below with the evidence.`, + `The overview, with every service, is at ${absolute(origin, "/compat/index.md")}.`, + ...(blocks.length ? ["## Tests Overcast does not yet pass", ...blocks] : []), + ...(otherLines.length ? ["## Not measured", "Results that say nothing about Overcast itself.", otherLines.join("\n")] : []), + ...(untested.length + ? ["## Not tested", "Operations with no test. The generator declined to write one, for the reason given.", untested.join("\n")] + : []), + ]); +} + +/** /compat/reason//index.md — every result with one reason, by service. */ +export function compatReasonMarkdown(code: string, { report, naming, origin }: CompatMarkdownOptions): string { + const reason = report.reasons.find((r) => r.code === code); + const label = reason?.label ?? code; + const labels = new Map(report.suites.map((suite) => [suite.id, suite.label])); + const blocks = report.services + .map((service) => { + const lines = service.groups.flatMap((group) => + group.tests + .map((test) => ({ + test, + suites: Object.entries(test.results) + .filter(([, result]) => result.reason === code) + .map(([suite]) => labels.get(suite) ?? suite), + })) + .filter(({ suites }) => suites.length) + .map(({ test, suites }) => `- ${test.name}${test.op !== test.name ? ` (${test.op})` : ""}: ${suites.join(", ")}`), + ); + const gaps = code === "untested" ? (service.untested ?? []).map((gap) => `- ${gap.op}: ${gap.detail}`) : []; + const all = [...lines, ...gaps]; + return all.length ? [`## ${serviceDisplayName(service.id, naming)}`, "", ...all].join("\n") : ""; + }) + .filter(Boolean); + + return page(`${label}: Overcast ${report.version} compatibility`, `> ${reason?.summary ?? label}`, [ + `${reasonCount(report, code)} ${code === "untested" ? "operations" : "results"}. The overview is at ${absolute(origin, "/compat/index.md")}.`, + ...blocks, + ]); +} + +/** /compat/explore/index.md — the explorer is interactive; its twin says where the data is. */ +export function compatExploreMarkdown({ origin }: MarkdownPageOptions): string { + return page("Explore compatibility results", "> Search and filter every Overcast compatibility test result.", [ + `The page filters one compact index per release, served at ${absolute(origin, "/compat/data/")}.json. The full report for a release is its \`compat-report.json\` release asset, and the overview is at ${absolute(origin, "/compat/index.md")}.`, + ]); +} diff --git a/src/lib/site-pages.ts b/src/lib/site-pages.ts index 6c6cea3..d3f75e3 100644 --- a/src/lib/site-pages.ts +++ b/src/lib/site-pages.ts @@ -36,6 +36,17 @@ export const SITE_PAGES: readonly SitePage[] = [ title: "Support matrix", description: "Overcast service and operation implementation status.", }, + { + path: "/compat/", + title: "Compatibility report", + description: + "Overcast's compatibility test results across six AWS SDKs, the AWS CLI and the CDK, with the reason for every result that does not pass.", + }, + { + path: "/compat/explore/", + title: "Explore compatibility results", + description: "Search and filter every Overcast compatibility test result by operation, client, reason and tracking issue.", + }, { path: "/downloads/", title: "Downloads", diff --git a/src/pages/[...slug].md.ts b/src/pages/[...slug].md.ts index 8fd7aaf..e0371f3 100644 --- a/src/pages/[...slug].md.ts +++ b/src/pages/[...slug].md.ts @@ -24,7 +24,13 @@ import { getServiceSupport } from "../lib/generated-content"; import { repositoryUrl, websiteGitHubRepo } from "../lib/github-links"; import { newestReleaseWithAssets, sortReleasesNewestFirst } from "../lib/releases"; import { LLMS_SECTION_ORDER } from "../lib/llms-txt"; +import { getCompatIndex, getCompatReport, getServiceNaming } from "../lib/compat-data"; +import { reasonsInUse } from "../lib/compat-report"; import { + compatExploreMarkdown, + compatOverviewMarkdown, + compatReasonMarkdown, + compatServiceMarkdown, consoleMarkdown, contributingMarkdown, docsIndexMarkdown, @@ -144,7 +150,34 @@ export const getStaticPaths: GetStaticPaths = async () => { }, ]; - return [...docPaths, ...sitePaths]; + // The compatibility report: the newest release at /compat/ with its service and reason + // pages, and each older one at /compat/history// — the same set the .astro routes build. + const compatIndex = await getCompatIndex(); + const naming = await getServiceNaming(); + const compatPaths: typeof sitePaths = []; + for (const [i, entry] of compatIndex.entries()) { + const report = await getCompatReport(entry.tag); + if (!report) continue; + const compat = { ...options, report, naming, latest: i === 0 }; + if (i > 0) { + compatPaths.push({ params: { slug: `compat/history/${entry.tag}/index` }, props: { body: compatOverviewMarkdown(compat) } }); + continue; + } + compatPaths.push( + { params: { slug: "compat/index" }, props: { body: compatOverviewMarkdown(compat) } }, + { params: { slug: "compat/explore/index" }, props: { body: compatExploreMarkdown(options) } }, + ...report.services.map((service) => ({ + params: { slug: `compat/${service.id}/index` }, + props: { body: compatServiceMarkdown(service.id, compat) }, + })), + ...reasonsInUse(report).map((reason) => ({ + params: { slug: `compat/reason/${reason.code}/index` }, + props: { body: compatReasonMarkdown(reason.code, compat) }, + })), + ); + } + + return [...docPaths, ...sitePaths, ...compatPaths]; }; export const GET: APIRoute = ({ props }) => { diff --git a/src/pages/compat/[service].astro b/src/pages/compat/[service].astro new file mode 100644 index 0000000..4e68d78 --- /dev/null +++ b/src/pages/compat/[service].astro @@ -0,0 +1,433 @@ +--- +// One service's compatibility results for the newest release: every test, every client, and +// for each result that does not pass, the reason and the evidence. Complete without script; +// the filter bar only hides rows. Anchors (#--) are stable across releases, so a +// link to a test keeps working after the next one ships. +import type { GetStaticPaths } from "astro"; +import { ArrowLeft, BookOpen, ListFilter, Search, ShieldCheck, X } from "@lucide/astro"; +import SiteLayout from "../../components/SiteLayout.astro"; +import ServiceIcon from "../../components/ServiceIcon.astro"; +import Chip from "../../components/Chip.astro"; +import CompatCell from "../../components/CompatCell.astro"; +import CompatIconSprite from "../../components/CompatIconSprite.astro"; +import CompatResultDetail from "../../components/CompatResultDetail.astro"; +import { getLatestCompatRelease, getServiceNaming } from "../../lib/compat-data"; +import { + matrixRow, + serviceMeasured, + percent, + rateTone, + reasonByCode, + serviceDisplayName, + serviceDocPath, + suiteShort, + supportServiceId, + groupResults, + testAnchor, + trimError, + whoseOf, +} from "../../lib/compat-report"; + +export const getStaticPaths: GetStaticPaths = async () => { + const release = await getLatestCompatRelease(); + return (release?.report.services ?? []).map((service) => ({ params: { service: service.id } })); +}; + +const { service: serviceId } = Astro.params as { service: string }; +const [release, naming] = await Promise.all([getLatestCompatRelease(), getServiceNaming()]); +const report = release!.report; +const service = report.services.find((s) => s.id === serviceId)!; +const reasons = reasonByCode(report); +const name = serviceDisplayName(service.id, naming); +const docPath = serviceDocPath(service.id, naming); +const suites = report.suites.filter((suite) => service.groups.some((g) => g.tests.some((t) => t.results[suite.id]))); +const cells = matrixRow(service, suites, reasons); +const headline = serviceMeasured(service, suites, reasons); +const testCount = service.groups.reduce((sum, group) => sum + group.tests.length, 0); +const toneVar = { success: "--oc-success", warning: "--oc-warning", danger: "--oc-danger", muted: "--oc-muted", accent: "--oc-accent" } as const; + +// What the filter script needs to know about each reason: only whether it is Overcast's. +const overcastCodes = report.reasons.filter((reason) => reason.whose === "overcast").map((reason) => reason.code); +const showOptions = [ + { value: "all", label: "All results" }, + { value: "failing", label: "Not passing" }, + { value: "overcast", label: "Overcast gaps" }, +]; +--- + + + +
+ + Compatibility report {report.version} + +

+ + {name} +

+

+ {testCount} tests across {suites.length} {suites.length === 1 ? "client" : "clients"}. Of the {headline.ran} results that measure + Overcast, {headline.pass} pass ({percent(headline.pass, headline.ran)}%). +

+
+ { + docPath && ( + + + {name} reference + + ) + } + + Support matrix + + + Explore with filters + +
+ +
    + { + cells.map((cell, i) => ( +
  • + + {suiteShort(suites[i].id)} + + + {cell.ran ? ( + <> + {cell.pass} + /{cell.ran} + + ) : ( + no test + )} + passing in {suites[i].label} + +
    • + )) + } +
+ +
+
+ + + + +
+
+ { + showOptions.map((option) => ( + + )) + } +
+ +
+

+ {testCount} tests +

+ + { + service.groups.map((group) => ( +
+

+ + {group.id} + + {group.generated && generated from the AWS model} + {group.state === "candidate" && soaking} +

+
+ + + + + {suites.map((suite) => ( + + ))} + + + + {group.tests.map((test) => { + const anchor = testAnchor(group.id, test.name); + const failing = suites.filter((suite) => test.results[suite.id]?.reason); + const cellsAttr = suites.map((suite) => `${suite.id}=${test.results[suite.id] ? (test.results[suite.id].reason ?? "pass") : ""}`).join(";"); + const errors = Object.values(test.results) + .map((r) => [trimError(r.error), r.issue ? `#${r.issue.number}` : ""].join(" ")) + .join(" "); + const overcastFailing = failing.filter((suite) => whoseOf(test.results[suite.id].reason!, reasons) === "overcast").length; + return ( + <> + + + {suites.map((suite) => ( + + ))} + + {failing.length > 0 && ( + + + + )} + + ); + })} + +
Test + + {suiteShort(suite.id)} + +
+ + {test.name} + + {test.op !== test.name && {test.op}} + {test.requires?.length ? needs {test.requires.join(", ")} : null} + + +
+
0 ? "" : undefined}> + + Why {failing.length === 1 ? "1 result does" : `${failing.length} results do`} not pass + {overcastFailing > 0 && · {overcastFailing} on Overcast} + +
+ {groupResults(test, suites, reasons).map((shared) => ( +
entry.suite.id).join(" ")}> + +
+ ))} +
+
+
+
+
+ )) + } + + + + { + service.untested?.length ? ( +
+

+ Not tested ({service.untested.length}) +

+

+ Operations with no test at all. The test generator declined to write one, for the reason given. +

+
+ + + + + + + + + + {service.untested.map((gap) => ( + + + + + + ))} + +
OperationReasonDetail
+ {gap.op} + + {gap.code} + {gap.issue && ( + + #{gap.issue.number} + + )} + {gap.detail}
+
+
+ ) : null + } +
+ + + diff --git a/src/pages/compat/data/[tag].json.ts b/src/pages/compat/data/[tag].json.ts new file mode 100644 index 0000000..596b85f --- /dev/null +++ b/src/pages/compat/data/[tag].json.ts @@ -0,0 +1,18 @@ +import type { APIRoute, GetStaticPaths } from "astro"; +import { getCompatIndex, getCompatRelease, getServiceNaming } from "../../../lib/compat-data"; +import { buildExplorerIndex } from "../../../lib/compat-report"; + +// https://overcast.sh/compat/data/.json — the explorer's compact index for one release +// (see ExplorerIndex in src/lib/compat-report.ts). Fetched only by /compat/explore/, so the +// static pages never pay for it. The full report is the release asset itself. +export const getStaticPaths: GetStaticPaths = async () => { + const index = await getCompatIndex(); + return index.map((entry) => ({ params: { tag: entry.tag } })); +}; + +export const GET: APIRoute = async ({ params }) => { + const release = await getCompatRelease(params.tag as string); + if (!release) return new Response("not found", { status: 404 }); + const body = buildExplorerIndex(release.report, await getServiceNaming(), release.previous ?? undefined); + return new Response(JSON.stringify(body), { headers: { "Content-Type": "application/json; charset=utf-8" } }); +}; diff --git a/src/pages/compat/explore.astro b/src/pages/compat/explore.astro new file mode 100644 index 0000000..25161eb --- /dev/null +++ b/src/pages/compat/explore.astro @@ -0,0 +1,424 @@ +--- +// Every compatibility result for a release, filtered as you type. The data is one compact +// index per release (/compat/data/.json, see ExplorerIndex); what matches is decided by +// src/lib/compat-filter.ts, which is unit-tested, and this page only renders it. Every view is +// a URL: ?release=&q=&outcome=&whose=&suite=&issue=&changed=. +import { ArrowLeft, ListFilter, Search, X } from "@lucide/astro"; +import SiteLayout from "../../components/SiteLayout.astro"; +import { getCompatIndex } from "../../lib/compat-data"; + +const index = await getCompatIndex(); +const releases = index.map((entry) => entry.tag); +--- + + +
+ + Compatibility report + +

Explore every result

+

+ Search by operation, test, service, error message or issue number, then narrow by outcome, client and owner. Every + view is a link: copy the address to share exactly what you are looking at. +

+ + { + releases.length === 0 ? ( +

No compatibility report has been published yet.

+ ) : ( + <> +
+
+ + + + +
+ +
+ +
+ +
+
+

+ Loading results… +

+ +
+
+ +
+
+ + ) + } +
+ + + diff --git a/src/pages/compat/history/[tag].astro b/src/pages/compat/history/[tag].astro new file mode 100644 index 0000000..31bd56c --- /dev/null +++ b/src/pages/compat/history/[tag].astro @@ -0,0 +1,25 @@ +--- +// An earlier release's report. The newest release lives at /compat/ alone, so every URL here +// stays put when the next release ships. +import type { GetStaticPaths } from "astro"; +import SiteLayout from "../../../components/SiteLayout.astro"; +import CompatOverview from "../../../components/CompatOverview.astro"; +import { getCompatIndex, getCompatRelease, getServiceNaming } from "../../../lib/compat-data"; + +export const getStaticPaths: GetStaticPaths = async () => { + const index = await getCompatIndex(); + return index.slice(1).map((entry) => ({ params: { tag: entry.tag } })); +}; + +const { tag } = Astro.params as { tag: string }; +const [release, index, naming] = await Promise.all([getCompatRelease(tag), getCompatIndex(), getServiceNaming()]); +if (!release) throw new Error(`compat: no report for ${tag}`); +--- + + + + diff --git a/src/pages/compat/index.astro b/src/pages/compat/index.astro new file mode 100644 index 0000000..7012f51 --- /dev/null +++ b/src/pages/compat/index.astro @@ -0,0 +1,16 @@ +--- +import SiteLayout from "../../components/SiteLayout.astro"; +import CompatOverview from "../../components/CompatOverview.astro"; +import CompatEmpty from "../../components/CompatEmpty.astro"; +import { getCompatIndex, getLatestCompatRelease, getServiceNaming } from "../../lib/compat-data"; + +const [release, index, naming] = await Promise.all([getLatestCompatRelease(), getCompatIndex(), getServiceNaming()]); +--- + + + {release ? : } + diff --git a/src/pages/compat/reason/[code].astro b/src/pages/compat/reason/[code].astro new file mode 100644 index 0000000..51e4620 --- /dev/null +++ b/src/pages/compat/reason/[code].astro @@ -0,0 +1,157 @@ +--- +// Every result with one reason, for the newest release, grouped by service. The overview's +// reason cards link here, so each "See them all" is a complete static list that works without +// script and that site search can index. The explorer does the same with any combination of +// filters; this is the fixed, linkable answer to "which ones, and why". +import type { GetStaticPaths } from "astro"; +import { ArrowLeft, ListFilter } from "@lucide/astro"; +import SiteLayout from "../../../components/SiteLayout.astro"; +import ServiceIcon from "../../../components/ServiceIcon.astro"; +import Chip from "../../../components/Chip.astro"; +import CompatIconSprite from "../../../components/CompatIconSprite.astro"; +import CompatResultDetail from "../../../components/CompatResultDetail.astro"; +import { getLatestCompatRelease, getServiceNaming } from "../../../lib/compat-data"; +import { + WHOSE_LABELS, + WHOSE_TONE, + groupResults, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + suiteShort, + supportServiceId, + testAnchor, +} from "../../../lib/compat-report"; + +export const getStaticPaths: GetStaticPaths = async () => { + const release = await getLatestCompatRelease(); + return release ? reasonsInUse(release.report).map((reason) => ({ params: { code: reason.code } })) : []; +}; + +const { code } = Astro.params as { code: string }; +const [release, naming] = await Promise.all([getLatestCompatRelease(), getServiceNaming()]); +const report = release!.report; +const reason = report.reasons.find((r) => r.code === code)!; +const total = reasonCount(report, code); +const withEvidence = reason.whose === "overcast"; +const reasons = reasonByCode(report); + +// Tests that have this reason in at least one suite, by service. +const services = report.services + .map((service) => { + const name = serviceDisplayName(service.id, naming); + const tests = service.groups.flatMap((group) => + group.tests + .map((test) => ({ group: group.id, test, suites: report.suites.filter((suite) => test.results[suite.id]?.reason === code) })) + .filter((row) => row.suites.length > 0), + ); + const untested = code === "untested" ? (service.untested ?? []) : []; + return { service, name, tests, untested }; + }) + .filter((entry) => entry.tests.length > 0 || entry.untested.length > 0) + .sort((a, b) => a.name.localeCompare(b.name)); +--- + + + +
+ + Compatibility report {report.version} + +

{reason.label}

+

{reason.summary}

+
+ {WHOSE_LABELS[reason.whose].title} + + {total} {code === "untested" ? "operations" : "results"} in {services.length} {services.length === 1 ? "service" : "services"} + + + Narrow this down + +
+ + + + { + services.map(({ service, name, tests, untested }) => ( +
+

+ + + {name} + +

+ {tests.length > 0 && ( +
    + {tests.map(({ group, test, suites }) => ( +
  • +
    + + {test.name} + + {test.op !== test.name && {test.op}} + {group} + + {suites.map((suite) => ( + + {suiteShort(suite.id)} + + ))} + +
    + {withEvidence && ( +
    + Evidence +
    + {groupResults(test, suites, reasons).map((shared) => ( + + ))} +
    +
    + )} +
    • + ))} +
+ )} + {untested.length > 0 && ( +
    + {untested.map((gap) => ( +
  • + {gap.op} + + {gap.code} + + {gap.detail} +
    • + ))} +
+ )} +
+ )) + } +
+ diff --git a/src/pages/sitemap.xml.ts b/src/pages/sitemap.xml.ts index 05b6579..1daefa2 100644 --- a/src/pages/sitemap.xml.ts +++ b/src/pages/sitemap.xml.ts @@ -1,6 +1,8 @@ import type { APIRoute } from "astro"; import { getCollection, type CollectionEntry } from "astro:content"; import { SITE_PAGE_PATHS } from "../lib/site-pages"; +import { getCompatIndex, getLatestCompatRelease } from "../lib/compat-data"; +import { reasonsInUse } from "../lib/compat-report"; function escapeXml(value: string): string { return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); @@ -13,9 +15,20 @@ export const GET: APIRoute = async ({ site }) => { // same page in another format, and listing both would put two URLs per doc in front of a // crawler. /llms.txt is absent for the same reason — it indexes this list, it isn't a // page in it. robots.txt points at it instead. + // The compatibility report's generated pages: one per service and per reason for the newest + // release, and one overview per older release. /compat/ itself is in SITE_PAGE_PATHS. + const compat = await getLatestCompatRelease(); + const compatRoutes = compat + ? [ + ...compat.report.services.map((service) => `/compat/${service.id}/`), + ...reasonsInUse(compat.report).map((reason) => `/compat/reason/${reason.code}/`), + ...(await getCompatIndex()).slice(1).map((entry) => `/compat/history/${entry.tag}/`), + ] + : []; const routes = [ ...SITE_PAGE_PATHS, ...docs.map((doc) => `/${doc.data.slug}/`), + ...compatRoutes, ].sort(); const urls = routes diff --git a/src/styles/global.css b/src/styles/global.css index fc0abcf..37c70c0 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -766,6 +766,159 @@ kbd { } +/* Compatibility report (/compat/). A cell is one suite's result for one test: an icon and a + * word on a tint, the same colour-mix-into-card pairing as so every tone clears 4.5:1 + * on either theme. CompatCell and the explorer set `data-tone`. */ +.compat-cell { + --cell-tone: var(--oc-muted); + --cell-bg: color-mix(in srgb, var(--cell-tone) 14%, var(--oc-card)); + --cell-fg: var(--cell-tone); + display: inline-flex; + align-items: center; + gap: 0.3rem; + min-width: 4.25rem; + padding: 0.125rem 0.4rem; + border-radius: 4px; + background: var(--cell-bg); + color: var(--cell-fg); + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + line-height: 1.25rem; + white-space: nowrap; +} + +.compat-cell[data-tone="success"] { --cell-tone: var(--oc-success); } +.compat-cell[data-tone="warning"] { --cell-tone: var(--oc-warning); } +.compat-cell[data-tone="danger"] { --cell-tone: var(--oc-danger); } +.compat-cell[data-tone="accent"] { --cell-tone: var(--oc-accent); } +/* As on : the grey tint keeps its hue, and the label steps up to --oc-body to clear 4.5:1. */ +.compat-cell[data-tone="muted"] { --cell-fg: var(--oc-body); } + +.compat-cell .icon { + width: 0.8125rem; + height: 0.8125rem; + flex-shrink: 0; +} + +.compat-cell-empty { + justify-content: center; + background: transparent; + color: var(--oc-muted); +} + +/* The matrix and the per-service tables scroll sideways on a phone; the first column (the + * service or the test) stays pinned so a row never loses its name. */ +.compat-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + font-size: 0.875rem; +} + +.compat-table th, +.compat-table td { + padding: 0.5rem 0.625rem; + border-bottom: 1px solid var(--oc-line); + text-align: left; + vertical-align: middle; +} + +.compat-table thead th { + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + font-weight: 400; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--oc-muted); + white-space: nowrap; +} + +.compat-table tbody tr:last-child > * { + border-bottom: 0; +} + +.compat-table tbody tr[data-detail] > td { + padding-top: 0; +} + +.compat-table :is(th, td):first-child { + position: sticky; + left: 0; + z-index: 1; + background: var(--oc-card); + box-shadow: 1px 0 0 var(--oc-line); +} + +.compat-table tbody tr:hover > :is(th, td) { + background: color-mix(in srgb, var(--oc-accent-soft) 60%, var(--oc-card)); +} + +.compat-table tr:target > :is(th, td) { + background: var(--oc-accent-soft); +} + +/* A matrix cell: pass/ran over a hairline meter tinted by the rate. */ +.compat-rate { + display: flex; + flex-direction: column; + gap: 0.25rem; + min-width: 3.75rem; + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + color: var(--oc-body); + text-decoration: none; +} + +.compat-rate .coverage-meter > span { + background: var(--rate-tone, var(--oc-accent)); +} + +/* Facet toggles on the explorer and the service pages: the support matrix's pressed-button + * pattern, with a count after the label. */ +.compat-toggle { + display: inline-flex; + align-items: center; + gap: 0.375rem; + padding: 0.3rem 0.6rem; + border: 1px solid var(--oc-line); + border-radius: var(--oc-radius-control); + background: var(--oc-card); + color: var(--oc-body); + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + cursor: pointer; +} + +.compat-toggle:hover { + background: var(--oc-accent-soft); +} + +.compat-toggle[aria-pressed="true"] { + border-color: var(--oc-accent); + background: var(--oc-accent-soft); + color: var(--oc-text); +} + +/* A facet that would leave nothing. Said with a dashed outline rather than by fading: faded + * text at 12px falls under 4.5:1 in every theme. */ +.compat-toggle[data-empty] { + border-style: dashed; + background: transparent; +} + +/* An explorer cell outside the current filter: kept for context, set back the same way. */ +.compat-cell[data-dim] { + --cell-bg: transparent; + --cell-fg: var(--oc-body); + outline: 1px dashed var(--oc-line); + outline-offset: -1px; +} + +.compat-toggle .compat-count { + opacity: 0.75; +} + + @font-face { font-family: "JetBrains Mono"; src: url("/fonts/JetBrainsMono-Regular.ttf") format("truetype");