diff --git a/.env.example b/.env.example index df13626..1632071 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,15 @@ OVERCAST_REPO=overcast-sh/overcast # OVERCAST_SOURCE_REF=v0.0.1-alpha.38 OVERCAST_EDIT_REF=main +# Compatibility report (/compat/). The sync copies each release's compat-report.json asset. +# To preview a report built from a working copy (`go run ./cmd/compat --publish-report +# compat-report.json` upstream), point at it; several paths, newest first, separated by the +# platform's path delimiter (; on Windows, : elsewhere), also preview the release-over-release +# changes. It defaults to /compat-report.json when that file exists. +# OVERCAST_COMPAT_REPORT=/path/to/compat-report.json +# Set to 1 to skip listing release reports on GitHub (offline, or local reports only). +# OVERCAST_COMPAT_OFFLINE=1 + # GitHub edit links for website-owned pages. WEBSITE_REPO=overcast-sh/website WEBSITE_EDIT_REF=main diff --git a/AGENTS.md b/AGENTS.md index 97fcb7b..fe728aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,24 @@ Gotchas: - Edit-link behavior (`EDIT_LINK_MODE`, `OVERCAST_EDIT_REF`, `WEBSITE_EDIT_REF`, etc.) is also driven by env vars in `.env.example` — see `src/lib/github-links.ts`. +### The compatibility report (/compat/) + +`syncCompat()` in the same script downloads the `compat-report.json` asset from each of the +last twelve Overcast releases that have one (GitHub API, `GITHUB_TOKEN` when set) into +`src/generated/compat/.json`, with `index.json` listing them newest first. Upstream writes +the file with `go run ./cmd/compat --publish-report`; its schema is `compat/report.schema.json` +there, and `src/lib/compat-report.ts` mirrors it. A report in another schema version is skipped. +Nothing in this step can fail the build: with no report, `/compat/` renders an empty state. + +- Pages: `/compat/` (newest release), `/compat//`, `/compat/reason//`, + `/compat/explore/` and `/compat/history//` for older releases. `/compat/data/.json` + is the explorer's compact index, built from the report at build time. +- Every number on those pages comes from the pure helpers in `src/lib/compat-report.ts`, and the + explorer's matching from `src/lib/compat-filter.ts`; both are covered by + `src/lib/compat-report.test.ts`. Change a number there, never in a page. +- Local preview: `OVERCAST_COMPAT_REPORT` (see `.env.example`) points the sync at a report built + from a working copy, and `OVERCAST_COMPAT_OFFLINE=1` skips the GitHub listing. + ## Repo layout highlights - `src/pages/` — route entry points (`index.astro`, `docs/[...slug].astro`, diff --git a/scripts/sync-overcast-content.ts b/scripts/sync-overcast-content.ts index 4685c5e..8f8f327 100644 --- a/scripts/sync-overcast-content.ts +++ b/scripts/sync-overcast-content.ts @@ -7,6 +7,7 @@ import { walk, warnMissingAllowlistedDocs, } from "../src/lib/overcast-doc-allowlist.ts"; +import { SUPPORTED_SCHEMA as COMPAT_SCHEMA } from "../src/lib/compat-report.ts"; const repo = process.env.OVERCAST_REPO || "overcast-sh/overcast"; const sourceRef = process.env.OVERCAST_SOURCE_REF || process.env.OVERCAST_TRACKING_REF || "alpha"; @@ -159,6 +160,97 @@ async function syncSupport(sourceRoot: string): Promise { return services; } +// The compatibility report (/compat/). Every Overcast release attaches a compat-report.json, +// written by `go run ./cmd/compat --publish-report` upstream; this copies one per release into +// src/generated/compat/.json and lists them, newest first, in index.json. Older releases +// stay browsable, and the newest two give the overview its release-over-release changes. +// +// Nothing here is allowed to fail the build: a network hiccup, a release without the asset, or +// a report in a schema this site does not know all degrade to fewer (or no) reports, and the +// pages render an empty state. A local file wins over the releases, so a report generated from +// a working copy can be previewed before any release carries one. +const COMPAT_ASSET = "compat-report.json"; +// Enough history to see a trend, few enough that the build does not grow with every release. +const COMPAT_RELEASES_KEPT = 12; + +type CompatIndexEntry = { tag: string; publishedAt: string | null; prerelease: boolean; source: "release" | "local" }; + +async function readCompatReport(filePath: string): Promise<{ version?: string; schema?: number } | null> { + const report = await readJsonIfExists<{ version?: string; schema?: number } | null>(filePath, null); + if (!report || report.schema !== COMPAT_SCHEMA) return null; + return report; +} + +async function fetchReleaseCompatReports(outDir: string): Promise { + const headers: Record = { Accept: "application/vnd.github+json", "User-Agent": "overcast-website-sync" }; + if (process.env.GITHUB_TOKEN) headers.Authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + const response = await fetch(`https://api.github.com/repos/${repo}/releases?per_page=50`, { headers }); + if (!response.ok) throw new Error(`GitHub releases: HTTP ${response.status}`); + const releases = (await response.json()) as Array<{ + tag_name: string; + published_at: string | null; + prerelease: boolean; + draft?: boolean; + assets: Array<{ name: string; url: string }>; + }>; + + const entries: CompatIndexEntry[] = []; + for (const release of releases) { + if (release.draft || entries.length >= COMPAT_RELEASES_KEPT) continue; + const asset = release.assets.find((candidate) => candidate.name === COMPAT_ASSET); + if (!asset) continue; + // The API URL with an octet-stream Accept, rather than browser_download_url: it takes the + // same token, so a private fork or a rate-limited runner still gets the file. + const download = await fetch(asset.url, { headers: { ...headers, Accept: "application/octet-stream" } }); + if (!download.ok) { + console.warn(`compat: ${release.tag_name}: ${COMPAT_ASSET} returned HTTP ${download.status}; skipped`); + continue; + } + const text = await download.text(); + const target = path.join(outDir, `${release.tag_name}.json`); + await fs.writeFile(target, text, "utf8"); + if (!(await readCompatReport(target))) { + console.warn(`compat: ${release.tag_name}: ${COMPAT_ASSET} is not schema ${COMPAT_SCHEMA}; skipped`); + await fs.rm(target, { force: true }); + continue; + } + entries.push({ tag: release.tag_name, publishedAt: release.published_at, prerelease: release.prerelease, source: "release" }); + } + return entries; +} + +async function syncCompat(sourceRoot: string): Promise { + const outDir = path.join(generatedDir, "compat"); + await fs.rm(outDir, { recursive: true, force: true }); + await fs.mkdir(outDir, { recursive: true }); + + // OVERCAST_COMPAT_REPORT takes one path, or several separated by the platform's path + // delimiter, newest first — two local reports are enough to preview the release-over-release + // changes before two releases carry one. + let entries: CompatIndexEntry[] = []; + const localPaths = process.env.OVERCAST_COMPAT_REPORT + ? process.env.OVERCAST_COMPAT_REPORT.split(path.delimiter).filter(Boolean) + : [path.join(sourceRoot, COMPAT_ASSET)]; + for (const localPath of localPaths) { + const local = await readCompatReport(localPath); + if (!local?.version || entries.some((entry) => entry.tag === local.version)) continue; + await fs.copyFile(localPath, path.join(outDir, `${local.version}.json`)); + entries.push({ tag: local.version, publishedAt: null, prerelease: false, source: "local" }); + console.log(`compat: using the local report at ${localPath} (${local.version})`); + } + if (process.env.OVERCAST_COMPAT_OFFLINE !== "1") { + try { + const released = await fetchReleaseCompatReports(outDir); + entries = [...entries, ...released.filter((entry) => !entries.some((e) => e.tag === entry.tag))]; + } catch (error) { + console.warn(`compat: could not list release reports (${(error as Error).message}); /compat/ shows ${entries.length ? "the local report only" : "an empty state"}`); + } + } + + await writeJson(path.join(outDir, "index.json"), entries); + return entries.length; +} + // The vendored brand assets: [path inside the branding repo, path inside this repo]. // // These stay *tracked* in git even though a script writes them. The deploy workflow never @@ -254,6 +346,7 @@ async function main() { if (brandingRoot) await syncBranding(brandingRoot); const docsCount = await countDocs(sourceRoot); const services = await syncSupport(sourceRoot); + const compatReports = await syncCompat(sourceRoot); await writeJson(path.join(generatedDir, "source-manifest.json"), { repo, @@ -263,6 +356,7 @@ async function main() { generatedAt: new Date().toISOString(), docsCount, serviceCount: services.length, + compatReports, }); } diff --git a/src/components/CompatCell.astro b/src/components/CompatCell.astro new file mode 100644 index 0000000..13f6e63 --- /dev/null +++ b/src/components/CompatCell.astro @@ -0,0 +1,37 @@ +--- +// One suite's result for one test, as a compact chip: an icon and a word, tinted by how much +// it matters. Colour is never the only signal. The word is always there, and the screen-reader +// text states the suite and the reason in full. The icon is a of CompatIconSprite, which +// every page that renders cells includes once. +import { compatIconKey, reasonShort, reasonTone, type CompatReason, type CompatResult } from "../lib/compat-report"; + +interface Props { + result: CompatResult | undefined; + reasons: CompatReason[]; + /** For the screen-reader text: "AWS SDK for Go v2". */ + suiteLabel: string; +} + +const { result, reasons, suiteLabel } = Astro.props; +const code = result?.reason; +const reason = code ? reasons.find((r) => r.code === code) : undefined; +const tone = reasonTone(code, reasons); +const spoken = !result ? `${suiteLabel}: does not run this test` : !code ? `${suiteLabel}: passes` : `${suiteLabel}: ${reason?.label ?? code}`; +--- + +{ + result ? ( + + + + {spoken} + + ) : ( + + + {spoken} + + ) +} diff --git a/src/components/CompatEmpty.astro b/src/components/CompatEmpty.astro new file mode 100644 index 0000000..749f057 --- /dev/null +++ b/src/components/CompatEmpty.astro @@ -0,0 +1,15 @@ +--- +// /compat/ before any release has published a report: say so, and point at what does exist. +import { ClipboardCheck } from "@lucide/astro"; +--- + +
+

+ compatibility report +

+

No compatibility report has been published yet.

+

+ Each release attaches one once its compatibility run has finished. Until then, the + support matrix lists which operations Overcast implements. +

+
diff --git a/src/components/CompatIconSprite.astro b/src/components/CompatIconSprite.astro new file mode 100644 index 0000000..eb4ae10 --- /dev/null +++ b/src/components/CompatIconSprite.astro @@ -0,0 +1,34 @@ +--- +// The result icons, once per page. A service page shows up to a thousand result cells, and +// inlining a Lucide SVG in each came to a third of the page's bytes; every cell now points at +// one of these symbols with . Hidden from assistive tech: each cell states its result in +// words. +import { Ban, CircleCheck, CircleDashed, CircleX, Container, CornerDownRight, FilePlus, Hourglass, Minus, Shuffle } from "@lucide/astro"; +import { COMPAT_ICON_KEYS } from "../lib/compat-report"; + +const icons: Record<(typeof COMPAT_ICON_KEYS)[number], typeof CircleCheck> = { + pass: CircleCheck, + "behaviour-mismatch": CircleX, + "not-emulated": Ban, + "quarantined-flaky": Shuffle, + "dependency-failed": CornerDownRight, + "suite-not-written": FilePlus, + candidate: Hourglass, + "sdk-lacks-api": Minus, + "needs-environment": Container, + other: CircleDashed, +}; +--- + + diff --git a/src/components/CompatOverview.astro b/src/components/CompatOverview.astro new file mode 100644 index 0000000..309cacb --- /dev/null +++ b/src/components/CompatOverview.astro @@ -0,0 +1,406 @@ +--- +// The compatibility report's overview for one release: the headline numbers, what changed since +// the release before, every reason a result did not pass (grouped by who has to act), and the +// service × suite matrix. /compat/ renders it for the newest release and /compat/history// +// for older ones; `latest` decides whether rows link to the per-service pages, which exist for +// the newest release only. +import { ArrowRight, Braces, ClipboardCheck, ListFilter, Search, TrendingDown, TrendingUp, X } from "@lucide/astro"; +import Callout from "./Callout.astro"; +import Chip from "./Chip.astro"; +import ServiceIcon from "./ServiceIcon.astro"; +import { formatDate, toIsoDate } from "../lib/format-date"; +import type { CompatRelease } from "../lib/compat-data"; +import { + WHOSE_LABELS, + WHOSE_ORDER, + WHOSE_TONE, + countByWhose, + diffReports, + isFixed, + isRegressed, + matrixRow, + measured, + operationSummaries, + percent, + rateTone, + serviceMeasured, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + shortCommit, + suiteShort, + supportServiceId, + type CompatReportIndexEntry, + type ServiceNaming, +} from "../lib/compat-report"; + +interface Props { + release: CompatRelease; + naming: ServiceNaming; + index: CompatReportIndexEntry[]; + latest: boolean; +} + +const { release, naming, index, latest } = Astro.props; +const { report, previous, entry } = release; +const reasons = reasonByCode(report); +const repoUrl = `https://github.com/${report.issueRepo}`; +const releaseUrl = entry.source === "release" ? `${repoUrl}/releases/tag/${report.version}` : null; +const jsonUrl = `/compat/data/${encodeURIComponent(report.version)}.json`; +const rawUrl = entry.source === "release" ? `${repoUrl}/releases/download/${report.version}/compat-report.json` : null; +const exploreUrl = (query = "") => `/compat/explore/${latest ? "" : `?release=${encodeURIComponent(report.version)}`}${query ? `${latest ? "?" : "&"}${query}` : ""}`; +const reasonUrl = (code: string) => (latest ? `/compat/reason/${code}/` : exploreUrl(`outcome=${code}`)); + +const headline = measured(report.totals, reasons); +const ops = operationSummaries(report); +const verified = ops.filter((op) => op.verified).length; +const whoseCounts = countByWhose(report.totals, reasons); +const blocked = report.totals.byReason["dependency-failed"] ?? 0; +const toFix = headline.ran - headline.pass; +const published = entry.publishedAt ?? report.generatedAt; + +const changes = previous ? diffReports(previous, report) : []; +const fixed = changes.filter(isFixed).length; +const regressed = changes.filter(isRegressed).length; + +const inUse = reasonsInUse(report); +const byWhose = WHOSE_ORDER.map((whose) => ({ whose, reasons: inUse.filter((reason) => reason.whose === whose) })).filter( + (group) => group.reasons.length > 0, +); + +// Services sorted by name, each with its matrix row and its headline numbers. +const services = report.services + .map((service) => { + const cells = matrixRow(service, report.suites, reasons); + const m = serviceMeasured(service, report.suites, reasons); + return { service, name: serviceDisplayName(service.id, naming), cells, pass: m.pass, ran: m.ran, gaps: m.ran - m.pass }; + }) + .sort((a, b) => a.name.localeCompare(b.name)); + +const toneVar = { success: "--oc-success", warning: "--oc-warning", danger: "--oc-danger", muted: "--oc-line", accent: "--oc-accent" } as const; +--- + +
+

+ compatibility report +

+

Tested against eight AWS clients, with the reason for every gap.

+

+ Every release runs one shared set of API tests through six AWS SDKs, the AWS CLI and the CDK against Overcast. This is + the result for {report.version}: what passes, what does not, and why. +

+ +
+
+
+
release
+
+ {releaseUrl ? {report.version} : {report.version}} + {!latest && archived} +
+
+
+
run
+
+
+ { + report.commit && ( +
+
commit
+
+ + {shortCommit(report.commit)} + +
+
+ ) + } +
+
data
+
+ + compat-report.json + +
+
+
+ { + index.length > 1 && ( + + ) + } +
+ + {/* The same dt-before-dd, visually reversed, arrangement as the support matrix's tiles. */} +
+
+
results passing, of those that measure Overcast
+
+ {percent(headline.pass, headline.ran)}%{headline.pass}/{headline.ran} +
+
+
+
operations passing in at least one client
+
{verified}/{ops.length}
+
+
+
+ results Overcast has to fix{blocked > 0 && <>, plus {blocked} tests blocked behind them} +
+
{toFix}
+
+
+
operations with no test yet
+
{whoseCounts.none}
+
+
+ +
+ + A result is one test run through one client. The pass rate counts only results that say something about Overcast: + a test not yet written for a language, an operation an SDK cannot call, or a test blocked by an earlier failure is + listed under its reason and left out of the rate. + +
+ + { + previous && ( +
+

Since {previous.version}

+
+ + + {fixed} newly passing + + + + {regressed} no longer passing + +
+
+ ) + } + +

Why results do not pass

+

+ Every result that does not pass has exactly one reason. They are grouped by who has to act on them. +

+
+ { + byWhose.map(({ whose, reasons: group }) => ( +
+
+

{WHOSE_LABELS[whose].title}

+ {whoseCounts[whose]} +
+

{WHOSE_LABELS[whose].blurb}

+ +
+ )) + } +
+ +
+
+

By service and client

+

+ Each cell is passing results over results that measure Overcast, for that service in that client. + {latest ? "Open a service for every test and the reason behind each result." : "Open the explorer for every test in this release."} +

+
+ + Search every result + +
+ +
+
+ + + + +
+ +

+ {services.length} services +

+
+ +
+ + + + + + { + report.suites.map((suite) => ( + + )) + } + + + + { + services.map(({ service, name, cells, pass, ran, gaps }) => { + const href = latest ? `/compat/${service.id}/` : exploreUrl(`q=${encodeURIComponent(service.id)}`); + return ( + + + + {cells.map((cell, i) => { + const suite = report.suites[i]; + const others = Object.entries(cell.byReason) + .map(([code, n]) => `${n} ${reasons.get(code)?.label.toLowerCase() ?? code}`) + .join(", "); + const tone = rateTone(cell.pass, cell.ran); + return ( + + ); + })} + + ); + }) + } + +
ServiceRate + + {suiteShort(suite.id)} + +
+ + + {name} + + + {percent(pass, ran)}% + + {" "} + of {ran} results passing, {gaps} Overcast {gaps === 1 ? "gap" : "gaps"} + + + {cell.cells === 0 ? ( + + + {suite.label}: no tests for this service + + ) : ( + + + + )} +
+ +
+
+ + diff --git a/src/components/CompatResultDetail.astro b/src/components/CompatResultDetail.astro new file mode 100644 index 0000000..53d6da8 --- /dev/null +++ b/src/components/CompatResultDetail.astro @@ -0,0 +1,145 @@ +--- +// Why a test did not pass in one or more clients that share the same outcome: the reason in +// words, the evidence (a parsed expected/actual diff when the failure is a generated scenario's, +// the raw messages otherwise), what it is blocked by, and where it is tracked. Callers group +// clients by reason, evidence and issue first (groupResults), so "501 in all seven SDKs" reads once. +// +// Without a tracking issue, "Report this" opens a prefilled issue carrying the marker that links +// it back to this result on the next release. It is offered for Overcast's own gaps only: a test +// not yet written for a language is tracked upstream as parity debt, and an SDK limit is nobody's bug. +import { ExternalLink } from "@lucide/astro"; +import CompatCell from "./CompatCell.astro"; +import { suiteShort, testAnchor, type CompatReason, type CompatResult } from "../lib/compat-report"; +import { newIssueUrl } from "../lib/compat-issue-link"; + +interface Props { + /** One entry per client, all with the same reason. */ + entries: { suite: { id: string; label: string }; result: CompatResult }[]; + reasons: CompatReason[]; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + version: string; + issueRepo: string; +} + +const { entries, reasons, service, serviceName, group, test, op, version, issueRepo } = Astro.props; +const first = entries[0].result; +const reason = reasons.find((r) => r.code === first.reason); +// A soaking candidate is a real assertion failure that simply does not gate yet: its evidence +// and a way to report it matter as much as for any other mismatch. +const ownedByOvercast = (reason?.whose ?? "overcast") === "overcast" || first.reason === "candidate"; +const issue = entries.find((entry) => entry.result.issue)?.result.issue; +const m = first.mismatch; +const blockedBy = [...new Set(entries.flatMap((entry) => entry.result.blockedBy ?? []))]; +// The message is evidence for a failure. For a skip it is usually just the skip sentinel +// restating the reason, so it is shown only where it says something the label does not. +const showMessages = !blockedBy.length && (ownedByOvercast || first.reason === "other-skip" || first.reason === "needs-environment"); +const messages = showMessages ? entries.filter((entry) => entry.result.error) : []; +const sameMessage = messages.length > 1 && messages.every((entry) => entry.result.error === messages[0].result.error); +const clientNames = entries.map((entry) => entry.suite.label); +const reportUrl = + issue || !ownedByOvercast + ? null + : newIssueUrl({ + issueRepo, + service, + serviceName, + group, + test, + op, + suites: entries.map((entry) => entry.suite), + reasonLabel: reason?.label ?? first.reason ?? "", + error: messages[0]?.result.error, + version, + }); +--- + +
+
+ + 1 ? clientNames.join(", ") : undefined}> + {entries.length === 1 ? entries[0].suite.label : entries.map((entry) => suiteShort(entry.suite.id)).join(", ")} + + { + issue ? ( + + #{issue.number} + {issue.title && {issue.title}} + + ) : ( + reportUrl && ( + + Report this + + ) + ) + } +
+ {reason &&

{reason.summary}

} + { + blockedBy.length > 0 && ( +

+ Blocked by{" "} + {blockedBy.map((name, i) => ( + <> + {i > 0 && ", "} + + {name} + + + ))} + . +

+ ) + } + { + m && ( +
+
check
+
+ {m.kind} + {m.path && ( + <> + {" "} + at {m.path} + + )}{" "} + on {m.op} +
+
expected
+
{m.expected}
+
actual
+
{m.actual}
+
scenario
+
+ {m.scenario} {m.step} +
+
+ ) + } + { + messages.length > 0 && ( +
+ + {messages.length === 1 || sameMessage ? "Message" : `Messages (${messages.length})`} + +
+ {(sameMessage ? messages.slice(0, 1) : messages).map((entry) => ( +
+ {!sameMessage && messages.length > 1 &&

{entry.suite.label}

} + 
+                {entry.result.error}
+
+
+ ))} +
+
+ ) + } + {first.status === "unimplemented" && messages.length === 0 &&

HTTP 501 Not Implemented

} +
diff --git a/src/components/SiteLayout.astro b/src/components/SiteLayout.astro index e5a47d5..6a821da 100644 --- a/src/components/SiteLayout.astro +++ b/src/components/SiteLayout.astro @@ -1,6 +1,6 @@ --- import "../styles/global.css"; -import { BookOpen, Download, FileText, GitCompare, Menu, Monitor, PencilLine, RotateCcwClock, Search, ShieldCheck, Star, SunMoon, X } from "@lucide/astro"; +import { BookOpen, ClipboardCheck, Download, FileText, GitCompare, Menu, Monitor, PencilLine, RotateCcwClock, Search, ShieldCheck, Star, SunMoon, X } from "@lucide/astro"; import GitHubIcon from "./GitHubIcon.astro"; import { repositoryUrl, starsUrl, websiteEditUrlFromRoute } from "../lib/github-links"; import { getSourceManifest } from "../lib/generated-content"; @@ -65,6 +65,7 @@ const primaryNav = [ { href: "/downloads/", label: "Downloads", footerLabel: "Downloads", Icon: Download }, { href: "/releases/", label: "Releases", footerLabel: "Releases", Icon: RotateCcwClock }, { href: "/support/", label: "Support Matrix", footerLabel: "Support Matrix", Icon: ShieldCheck }, + { href: "/compat/", label: "Compatibility", footerLabel: "Compatibility Report", Icon: ClipboardCheck }, { href: "/docs/migration-from-localstack/", label: "LocalStack Migration", footerLabel: "LocalStack Migration", Icon: GitCompare }, ]; // `aria-current="page"` is the only thing that tells a screen reader which nav entry is diff --git a/src/lib/compat-data.ts b/src/lib/compat-data.ts new file mode 100644 index 0000000..7c43ccc --- /dev/null +++ b/src/lib/compat-data.ts @@ -0,0 +1,67 @@ +/** + * Build-time access to the synced compatibility reports (src/generated/compat/). The + * pure helpers live in compat-report.ts; this file only reads, and caches, because every + * service page and the explorer data route read the same few megabytes. + */ +import path from "node:path"; +import fs from "node:fs/promises"; +import { getServiceSupport } from "./generated-content"; +import type { CompatReport, CompatReportIndexEntry, ServiceNaming } from "./compat-report"; + +const compatRoot = path.join(process.cwd(), "src", "generated", "compat"); + +let indexPromise: Promise | undefined; +const reportPromises = new Map>(); + +/** Every synced release, newest first. Empty when no release has published a report yet. */ +export function getCompatIndex(): Promise { + indexPromise ??= fs + .readFile(path.join(compatRoot, "index.json"), "utf8") + .then((raw) => JSON.parse(raw) as CompatReportIndexEntry[]) + .catch(() => []); + return indexPromise; +} + +export function getCompatReport(tag: string): Promise { + let promise = reportPromises.get(tag); + if (!promise) { + promise = fs + .readFile(path.join(compatRoot, `${tag}.json`), "utf8") + .then((raw) => JSON.parse(raw) as CompatReport) + .catch(() => null); + reportPromises.set(tag, promise); + } + return promise; +} + +export interface CompatRelease { + entry: CompatReportIndexEntry; + report: CompatReport; + /** The release before this one, for release-over-release changes. */ + previous: CompatReport | null; +} + +/** A release and the one before it, by tag. */ +export async function getCompatRelease(tag: string): Promise { + const index = await getCompatIndex(); + const at = index.findIndex((entry) => entry.tag === tag); + if (at < 0) return null; + const report = await getCompatReport(tag); + if (!report) return null; + const previousEntry = index[at + 1]; + return { entry: index[at], report, previous: previousEntry ? await getCompatReport(previousEntry.tag) : null }; +} + +export async function getLatestCompatRelease(): Promise { + const [latest] = await getCompatIndex(); + return latest ? getCompatRelease(latest.tag) : null; +} + +/** Display names and doc links, from the support matrix the rest of the site uses. */ +export async function getServiceNaming(): Promise { + const support = await getServiceSupport(); + return { + displayNames: new Map(support.services.map((service) => [service.service, service.displayName])), + docSlugs: new Map(support.services.map((service) => [service.service, service.docSlug ?? service.service])), + }; +} diff --git a/src/lib/compat-filter.ts b/src/lib/compat-filter.ts new file mode 100644 index 0000000..c351053 --- /dev/null +++ b/src/lib/compat-filter.ts @@ -0,0 +1,282 @@ +/** + * compat-filter — the explorer's filtering, with no DOM in it. + * + * /compat/explore/ fetches a release's ExplorerIndex and filters it as the reader types. + * Everything that decides *what matches* lives here so `npm test` can pin it: the URL + * round-trip that makes every view a link, token matching, facet counts, and ranking. + * The page script only renders what this returns. + * + * The unit of filtering is the cell (one test, one suite). A test row is shown when at + * least one of its cells passes every facet, and the facet counts are cells too, so the + * numbers on the buttons always add up to the results they would leave. + */ +import type { ExplorerIndex, ExplorerRow, Whose } from "./compat-report.ts"; + +/** An outcome is a reason code, or one of these two. */ +export const PASS = "pass"; +export const UNTESTED = "untested"; + +export type IssueFilter = "any" | "linked" | "unlinked"; +export type ChangeFilter = "any" | "fixed" | "regressed"; + +export interface FilterState { + q: string; + /** Empty means every value; otherwise the selected ones, OR-ed together. */ + outcomes: string[]; + whose: Whose[]; + suites: string[]; + issue: IssueFilter; + changed: ChangeFilter; +} + +export const EMPTY_STATE: FilterState = { q: "", outcomes: [], whose: [], suites: [], issue: "any", changed: "any" }; + +const WHOSE_VALUES: readonly Whose[] = ["overcast", "suite", "sdk", "environment", "none"]; +const ISSUE_VALUES: readonly IssueFilter[] = ["any", "linked", "unlinked"]; +const CHANGE_VALUES: readonly ChangeFilter[] = ["any", "fixed", "regressed"]; + +const list = (value: string | null) => + (value ?? "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + +/** Reads the filter from a query string. Unknown values are dropped, never thrown on. */ +export function parseState(search: string): FilterState { + const params = new URLSearchParams(search); + const issue = params.get("issue") as IssueFilter | null; + const changed = params.get("changed") as ChangeFilter | null; + return { + q: params.get("q") ?? "", + outcomes: [...new Set(list(params.get("outcome")))], + whose: [...new Set(list(params.get("whose")))].filter((w): w is Whose => WHOSE_VALUES.includes(w as Whose)), + suites: [...new Set(list(params.get("suite")))], + issue: issue && ISSUE_VALUES.includes(issue) ? issue : "any", + changed: changed && CHANGE_VALUES.includes(changed) ? changed : "any", + }; +} + +/** + * Writes the filter back as query parameters, leaving out every default so an unfiltered + * view is the bare URL. `release` is carried through untouched: it picks the data, not the + * filter. + */ +export function serializeState(state: FilterState, release?: string): string { + const params = new URLSearchParams(); + if (release) params.set("release", release); + if (state.q.trim()) params.set("q", state.q.trim()); + if (state.outcomes.length) params.set("outcome", state.outcomes.join(",")); + if (state.whose.length) params.set("whose", state.whose.join(",")); + if (state.suites.length) params.set("suite", state.suites.join(",")); + if (state.issue !== "any") params.set("issue", state.issue); + if (state.changed !== "any") params.set("changed", state.changed); + const out = params.toString(); + return out ? `?${out}` : ""; +} + +export function isFiltered(state: FilterState): boolean { + return Boolean(state.q.trim()) || state.outcomes.length > 0 || state.whose.length > 0 || state.suites.length > 0 || state.issue !== "any" || state.changed !== "any"; +} + +// ── Preparation ────────────────────────────────────────────────────────────── + +/** "CloudWatch Logs", "cloudwatch-logs" and "cloudwatchlogs" all find the same thing. */ +export const squash = (value: string): string => value.toLowerCase().replace(/[^a-z0-9#]/g, ""); + +export interface PreparedRow { + row: ExplorerRow; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + /** Everything a word may match in, lower-cased and squashed. */ + haystack: string; + /** Cell keys for the changed-since-last-release facet, one per suite (or null). */ + keys: string[]; +} + +export interface PreparedUntested { + service: string; + serviceName: string; + op: string; + code: string; + detail: string; + issue: number; + haystack: string; +} + +export interface PreparedIndex { + index: ExplorerIndex; + rows: PreparedRow[]; + untested: PreparedUntested[]; + fixed: Set; + regressed: Set; +} + +export function prepare(index: ExplorerIndex): PreparedIndex { + const rows = index.rows.map((row): PreparedRow => { + const [s, g, test, opOrEmpty, , details] = row; + const service = index.services[s]; + const group = index.groups[g]; + const op = opOrEmpty || test; + const extras = Object.values(details).flatMap(([error, issue, blocked]) => [error, issue ? `#${issue}` : "", blocked]); + const haystack = [service.id, service.name, group, test, op, ...extras].map(squash).join(" "); + const keys = index.suites.map((suite) => `${service.id}/${group}/${test}@${suite.id}`); + return { row, service: service.id, serviceName: service.name, group, test, op, haystack, keys }; + }); + const untested = index.untested.map(([s, op, code, detail, issue]): PreparedUntested => { + const service = index.services[s]; + return { + service: service.id, + serviceName: service.name, + op, + code, + detail, + issue, + haystack: [service.id, service.name, op, code, detail, issue ? `#${issue}` : ""].map(squash).join(" "), + }; + }); + return { index, rows, untested, fixed: new Set(index.changed?.fixed ?? []), regressed: new Set(index.changed?.regressed ?? []) }; +} + +// ── Evaluation ─────────────────────────────────────────────────────────────── + +export interface FacetCounts { + outcome: Record; + whose: Record; + suite: Record; + issue: Record; + changed: Record; +} + +export interface MatchedRow { + prepared: PreparedRow; + /** Per suite: does this cell pass every facet. Non-matching cells render dimmed. */ + mask: boolean[]; + rank: number; +} + +export interface Evaluation { + rows: MatchedRow[]; + untested: PreparedUntested[]; + /** Matching cells, plus matching untested operations. */ + total: number; + counts: FacetCounts; +} + +/** The words of a query, squashed. Empty words (pure punctuation) are dropped. */ +export function queryWords(q: string): string[] { + return q + .trim() + .split(/\s+/) + .map(squash) + .filter(Boolean); +} + +function textRank(words: string[], p: { op: string; test?: string; service: string }): number { + // Exact operation or test name first, then a name that starts with the first word, then + // everything else in report order. Only the first word ranks; the rest only filter. + if (!words.length) return 0; + const first = words[0]; + const names = [squash(p.op), squash(p.test ?? p.op)]; + if (names.includes(first)) return 3; + if (names.some((name) => name.startsWith(first))) return 2; + if (squash(p.service) === first) return 1; + return 0; +} + +type Cell = { outcome: string; whose: Whose; suite: string; issue: boolean; key: string }; + +/** + * Filters the prepared index. For each facet, its own counts are computed with every + * *other* facet applied, which is what makes a count say "pressing this would leave N". + */ +export function evaluate(prepared: PreparedIndex, state: FilterState): Evaluation { + const { index } = prepared; + const words = queryWords(state.q); + const outcomeOf = (cell: number) => (cell === -1 ? PASS : index.reasons[cell]?.code ?? "unknown"); + const whoseOfCell = (cell: number): Whose => (cell === -1 ? "overcast" : index.reasons[cell]?.whose ?? "overcast"); + + const outcomeOk = (c: Pick) => !state.outcomes.length || state.outcomes.includes(c.outcome); + // A pass belongs to nobody: the owner facet narrows to gaps, so a pass never survives it. + const whoseOk = (c: Pick) => !state.whose.length || (c.outcome !== PASS && state.whose.includes(c.whose)); + const suiteOk = (c: Pick) => !state.suites.length || state.suites.includes(c.suite); + const issueOk = (c: Pick) => state.issue === "any" || (state.issue === "linked") === c.issue; + const changedOk = (c: Pick) => + state.changed === "any" || (state.changed === "fixed" ? prepared.fixed.has(c.key) : prepared.regressed.has(c.key)); + const checks = { outcome: outcomeOk, whose: whoseOk, suite: suiteOk, issue: issueOk, changed: changedOk }; + type Facet = keyof typeof checks; + const passesExcept = (c: Cell, skip: Facet | null) => + (Object.keys(checks) as Facet[]).every((facet) => facet === skip || checks[facet](c as never)); + + const counts: FacetCounts = { + outcome: {}, + whose: {}, + suite: {}, + issue: { any: 0, linked: 0, unlinked: 0 }, + changed: { any: 0, fixed: 0, regressed: 0 }, + }; + const bump = (record: Record, key: string) => (record[key] = (record[key] ?? 0) + 1); + + const rows: MatchedRow[] = []; + let total = 0; + for (const p of prepared.rows) { + if (!words.every((word) => p.haystack.includes(word))) continue; + const [, , , , cells, details] = p.row; + const mask = cells.map(() => false); + cells.forEach((value, s) => { + if (value === null) return; + const cell: Cell = { + outcome: outcomeOf(value), + whose: whoseOfCell(value), + suite: index.suites[s].id, + issue: Boolean(details[s]?.[1]), + key: p.keys[s], + }; + if (passesExcept(cell, "outcome")) bump(counts.outcome, cell.outcome); + if (cell.outcome !== PASS && passesExcept(cell, "whose")) bump(counts.whose, cell.whose); + if (passesExcept(cell, "suite")) bump(counts.suite, cell.suite); + if (passesExcept(cell, "issue")) { + counts.issue.any += 1; + bump(counts.issue, cell.issue ? "linked" : "unlinked"); + } + if (passesExcept(cell, "changed")) { + counts.changed.any += 1; + if (prepared.fixed.has(cell.key)) counts.changed.fixed += 1; + if (prepared.regressed.has(cell.key)) counts.changed.regressed += 1; + } + if (passesExcept(cell, null)) { + mask[s] = true; + total += 1; + } + }); + if (mask.some(Boolean)) rows.push({ prepared: p, mask, rank: textRank(words, p) }); + } + + // Untested operations have no suite and no result. They join the outcome and owner + // facets as "untested" / "none", and drop out as soon as a suite, issue link or change + // is asked for, since none of those can describe them. + const untested: PreparedUntested[] = []; + const noSuiteFacets = !state.suites.length && state.changed === "any"; + for (const u of prepared.untested) { + if (!words.every((word) => u.haystack.includes(word))) continue; + const linked = u.issue > 0; + const cell = { outcome: UNTESTED, whose: "none" as Whose, issue: linked }; + const issueMatch = state.issue === "any" || (state.issue === "linked") === linked; + if (noSuiteFacets && issueMatch && whoseOk(cell)) bump(counts.outcome, UNTESTED); + if (noSuiteFacets && issueMatch && outcomeOk(cell)) bump(counts.whose, "none"); + if (noSuiteFacets && outcomeOk(cell) && whoseOk(cell)) { + counts.issue.any += 1; + bump(counts.issue, linked ? "linked" : "unlinked"); + } + if (noSuiteFacets && issueMatch && outcomeOk(cell) && whoseOk(cell)) { + untested.push(u); + total += 1; + } + } + + rows.sort((a, b) => b.rank - a.rank); + untested.sort((a, b) => textRank(words, { op: b.op, service: b.service }) - textRank(words, { op: a.op, service: a.service })); + return { rows, untested, total, counts }; +} diff --git a/src/lib/compat-issue-link.ts b/src/lib/compat-issue-link.ts new file mode 100644 index 0000000..82cc5be --- /dev/null +++ b/src/lib/compat-issue-link.ts @@ -0,0 +1,48 @@ +/** + * The "Report this" link on a compatibility result with no tracking issue: a new-issue form, + * prefilled, carrying the marker (``) that + * scripts/compat-issues.py in overcast-sh/overcast reads. Filing it is enough for the next + * release's report to link every listed client's result to the issue; nobody edits a mapping file. + */ + +export interface IssueLinkInput { + issueRepo: string; + service: string; + serviceName: string; + group: string; + test: string; + op: string; + /** The clients that share this outcome. */ + suites: { id: string; label: string }[]; + reasonLabel: string; + error?: string; + version: string; +} + +/** Long enough to recognise the failure, short enough to keep the URL well under 8 KB. */ +const ERROR_LIMIT = 1500; + +/** The marker targets for one test in the given clients: service/group/test@suite. */ +export function issueMarkerTargets({ service, group, test, suites }: Pick): string[] { + return suites.map((suite) => `${service}/${group}/${test}@${suite.id}`); +} + +export function newIssueUrl(input: IssueLinkInput): string { + const { issueRepo, serviceName, op, suites, reasonLabel, error, version, group, test } = input; + const clients = suites.length === 1 ? suites[0].label : `${suites.length} clients`; + const title = `Compat: ${serviceName} ${op} in ${clients}: ${reasonLabel.toLowerCase()}`; + const message = error ? (error.length > ERROR_LIMIT ? `${error.slice(0, ERROR_LIMIT)}…` : error) : ""; + const body = [ + `**Service:** ${serviceName}`, + `**Operation:** ${op} (test \`${group}/${test}\`)`, + `**Clients:** ${suites.map((suite) => suite.label).join(", ")}`, + `**Release:** ${version}`, + `**Reported as:** ${reasonLabel}`, + "", + ...(message ? ["```", message, "```", ""] : []), + "", + ``, + ].join("\n"); + const params = new URLSearchParams({ title, body, labels: "compat" }); + return `https://github.com/${issueRepo}/issues/new?${params.toString()}`; +} diff --git a/src/lib/compat-report.test.ts b/src/lib/compat-report.test.ts new file mode 100644 index 0000000..b7522ff --- /dev/null +++ b/src/lib/compat-report.test.ts @@ -0,0 +1,313 @@ +// Unit tests for the compatibility report helpers and the explorer's filter. Run with +// `npm test`. Both modules are pure, so the fixture below is all either needs. + +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; + +import { + buildExplorerIndex, + countByWhose, + diffReports, + groupResults, + isFixed, + isRegressed, + matrixRow, + measured, + operationSummaries, + rateTone, + reasonByCode, + serviceDisplayName, + serviceDocPath, + testAnchor, + trimError, + type CompatReport, + type ServiceNaming, +} from "./compat-report.ts"; +import { EMPTY_STATE, evaluate, isFiltered, parseState, prepare, serializeState, type FilterState } from "./compat-filter.ts"; +import { issueMarkerTargets, newIssueUrl } from "./compat-issue-link.ts"; +import { compatOverviewMarkdown, compatReasonMarkdown, compatServiceMarkdown } from "./site-markdown.ts"; + +const REASONS: CompatReport["reasons"] = [ + { code: "behaviour-mismatch", label: "Behaves differently from AWS", summary: "…", whose: "overcast" }, + { code: "not-emulated", label: "Not emulated yet", summary: "…", whose: "overcast" }, + { code: "dependency-failed", label: "Blocked by an earlier step", summary: "…", whose: "overcast" }, + { code: "suite-not-written", label: "Test not written for this SDK", summary: "…", whose: "suite" }, + { code: "sdk-lacks-api", label: "Not in this SDK", summary: "…", whose: "sdk" }, + { code: "untested", label: "Not tested", summary: "…", whose: "none" }, +]; + +function report(version: string, overrides: Partial> = {}): CompatReport { + // go-sdk and rust-sdk over two S3 tests and one ELB test. `overrides` sets a cell's reason + // by "test@suite" ("" is a pass), so a second release can move individual cells. + const cell = (test: string, suite: string, fallback: string) => { + const reason = overrides[`${test}@${suite}`] ?? fallback; + return reason ? { status: "skip" as const, reason, error: `${reason} for ${test}` } : { status: "pass" as const }; + }; + return { + schema: 1, + version, + generatedAt: "2026-09-24T00:00:00Z", + issueRepo: "overcast-sh/overcast", + reasons: REASONS, + suites: [ + { id: "go-sdk", label: "AWS SDK for Go v2", kind: "sdk", language: "Go", totals: { cells: 3, pass: 2, byReason: {} } }, + { id: "rust-sdk", label: "AWS SDK for Rust", kind: "sdk", language: "Rust", totals: { cells: 3, pass: 1, byReason: {} } }, + ], + totals: { cells: 6, pass: 3, byReason: { "not-emulated": 1, "suite-not-written": 2 }, untested: 1 }, + services: [ + { + id: "s3", + totals: { cells: 4, pass: 2, byReason: {} }, + groups: [ + { + id: "s3-crud", + tests: [ + { name: "CreateBucket", op: "CreateBucket", results: { "go-sdk": cell("CreateBucket", "go-sdk", ""), "rust-sdk": cell("CreateBucket", "rust-sdk", "") } }, + { + name: "PutObjectTagged", + op: "PutObject", + results: { + "go-sdk": cell("PutObjectTagged", "go-sdk", "not-emulated"), + "rust-sdk": { ...cell("PutObjectTagged", "rust-sdk", "suite-not-written"), issue: { number: 42, url: "https://github.com/o/r/issues/42" } }, + }, + }, + ], + }, + ], + }, + { + id: "elastic-load-balancing", + totals: { cells: 2, pass: 1, byReason: {} }, + groups: [ + { + id: "elb-crud", + tests: [ + { name: "CreateLoadBalancer", op: "CreateLoadBalancer", results: { "go-sdk": cell("CreateLoadBalancer", "go-sdk", ""), "rust-sdk": cell("CreateLoadBalancer", "rust-sdk", "suite-not-written") } }, + ], + }, + ], + untested: [{ op: "DeleteLoadBalancer", group: "elb-crud", reason: "never-probe", code: "never-probe", detail: "destructive" }], + }, + ], + }; +} + +const naming: ServiceNaming = { + displayNames: new Map([ + ["s3", "S3"], + ["elbv2", "ELBv2"], + ]), + docSlugs: new Map([ + ["s3", "s3"], + ["elbv2", "elb"], + ]), +}; + +describe("compat-report", () => { + it("names services through the support matrix, aliases included", () => { + assert.equal(serviceDisplayName("s3", naming), "S3"); + assert.equal(serviceDisplayName("elastic-load-balancing", naming), "ELBv2"); + assert.equal(serviceDocPath("elastic-load-balancing", naming), "/docs/services/elb/"); + assert.equal(serviceDisplayName("servicediscovery", naming), "Cloud Map"); + assert.equal(serviceDocPath("cdk", naming), null); + assert.equal(serviceDisplayName("brand-new", naming), "brand-new"); + }); + + it("builds stable anchors", () => { + assert.equal(testAnchor("s3-crud", "PutObjectTagged"), "s3-crud--putobjecttagged"); + assert.equal(testAnchor("g", "Weird Name/1"), "g--weird-name-1"); + }); + + it("measures only the cells that say something about Overcast", () => { + const r = report("v1"); + const reasons = reasonByCode(r); + assert.deepEqual(measured(r.totals, reasons), { pass: 3, ran: 4 }); + // A blocked test never ran, so it is not charged a second time. + assert.deepEqual(measured({ ...r.totals, byReason: { ...r.totals.byReason, "dependency-failed": 5 } }, reasons), { pass: 3, ran: 4 }); + assert.deepEqual(countByWhose(r.totals, reasons), { overcast: 1, suite: 2, sdk: 0, environment: 0, none: 1 }); + }); + + it("rolls a service up per suite", () => { + const r = report("v1"); + const [go, rust] = matrixRow(r.services[0], r.suites, reasonByCode(r)); + assert.deepEqual(go, { suite: "go-sdk", pass: 1, ran: 2, cells: 2, byReason: { "not-emulated": 1 } }); + assert.deepEqual(rust, { suite: "rust-sdk", pass: 1, ran: 1, cells: 2, byReason: { "suite-not-written": 1 } }); + assert.equal(rateTone(go.pass, go.ran), "danger"); + assert.equal(rateTone(rust.pass, rust.ran), "success"); + assert.equal(rateTone(0, 0), "muted"); + }); + + it("counts an operation verified when any suite passes it", () => { + const ops = operationSummaries(report("v1")); + const put = ops.find((op) => op.op === "PutObject"); + assert.deepEqual(put, { service: "s3", op: "PutObject", verified: false, failing: true }); + assert.equal(ops.filter((op) => op.verified).length, 2); + }); + + it("diffs two releases cell by cell", () => { + const before = report("v1"); + const after = report("v2", { "PutObjectTagged@go-sdk": "", "CreateBucket@rust-sdk": "behaviour-mismatch" }); + const changes = diffReports(before, after); + assert.equal(changes.length, 2); + assert.deepEqual(changes.filter(isFixed).map((c) => `${c.test}@${c.suite}`), ["PutObjectTagged@go-sdk"]); + assert.deepEqual(changes.filter(isRegressed).map((c) => `${c.test}@${c.suite}`), ["CreateBucket@rust-sdk"]); + }); + + it("trims long errors to one line", () => { + assert.equal(trimError("a\n b"), "a b"); + assert.equal(trimError("x".repeat(10), 5), "xxxx…"); + assert.equal(trimError(undefined), ""); + }); + + it("encodes the explorer index compactly", () => { + const index = buildExplorerIndex(report("v2", { "PutObjectTagged@go-sdk": "" }), naming, report("v1")); + assert.equal(index.rows.length, 3); + const put = index.rows[1]; + assert.equal(put[2], "PutObjectTagged"); + assert.equal(put[3], "PutObject"); + assert.deepEqual(put[4], [-1, 3]); + assert.deepEqual(put[5], { 1: ["suite-not-written for PutObjectTagged", 42, ""] }); + // An op equal to its test name is not repeated. + assert.equal(index.rows[0][3], ""); + assert.deepEqual(index.untested, [[1, "DeleteLoadBalancer", "never-probe", "destructive", 0]]); + assert.equal(index.services[1].name, "ELBv2"); + assert.deepEqual(index.changed, { previous: "v1", fixed: ["s3/s3-crud/PutObjectTagged@go-sdk"], regressed: [] }); + }); + + it("keeps a reason code the report does not describe", () => { + const r = report("v1", { "CreateBucket@go-sdk": "brand-new-reason" }); + const index = buildExplorerIndex(r, naming); + const code = index.rows[0][4][0] as number; + assert.equal(index.reasons[code].code, "brand-new-reason"); + assert.equal(index.reasons.length, REASONS.length + 1); + }); +}); + +describe("compat-filter", () => { + const prepared = prepare(buildExplorerIndex(report("v2", { "PutObjectTagged@go-sdk": "" }), naming, report("v1"))); + const run = (state: Partial) => evaluate(prepared, { ...EMPTY_STATE, ...state }); + + it("round-trips the URL, dropping defaults and junk", () => { + const state: FilterState = { q: "put object", outcomes: ["not-emulated", "pass"], whose: ["overcast"], suites: ["rust-sdk"], issue: "linked", changed: "fixed" }; + const query = serializeState(state, "v2"); + assert.equal(query, "?release=v2&q=put+object&outcome=not-emulated%2Cpass&whose=overcast&suite=rust-sdk&issue=linked&changed=fixed"); + assert.deepEqual(parseState(query), state); + assert.equal(serializeState(EMPTY_STATE), ""); + assert.deepEqual(parseState("?whose=nobody&issue=maybe&changed=sideways"), EMPTY_STATE); + assert.equal(isFiltered(EMPTY_STATE), false); + assert.equal(isFiltered({ ...EMPTY_STATE, q: " " }), false); + }); + + it("shows every row and counts every cell when nothing is filtered", () => { + const result = run({}); + assert.equal(result.rows.length, 3); + assert.equal(result.untested.length, 1); + assert.equal(result.total, 7); + assert.deepEqual(result.counts.outcome, { pass: 4, "suite-not-written": 2, untested: 1 }); + assert.equal(result.counts.suite["go-sdk"], 3); + }); + + it("matches every word, across names, errors and issue numbers", () => { + assert.deepEqual(run({ q: "put object" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + assert.deepEqual(run({ q: "elbv2" }).rows.map((r) => r.prepared.test), ["CreateLoadBalancer"]); + assert.deepEqual(run({ q: "#42" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + assert.equal(run({ q: "put nothing-like-this" }).rows.length, 0); + }); + + it("ranks an exact operation name first", () => { + const rows = run({ q: "createloadbalancer" }).rows; + assert.equal(rows[0].prepared.test, "CreateLoadBalancer"); + }); + + it("combines facets, and masks the cells that do not match", () => { + const result = run({ outcomes: ["suite-not-written"], suites: ["rust-sdk"] }); + assert.deepEqual(result.rows.map((r) => r.prepared.test), ["PutObjectTagged", "CreateLoadBalancer"]); + assert.deepEqual(result.rows[0].mask, [false, true]); + assert.equal(result.untested.length, 0, "a suite filter cannot describe an untested operation"); + }); + + it("counts each facet with the others applied", () => { + const result = run({ suites: ["rust-sdk"] }); + // Outcome counts follow the suite filter... + assert.deepEqual(result.counts.outcome, { pass: 1, "suite-not-written": 2 }); + // ...and the suite counts ignore it, so the other suite's button says what it would show. + assert.equal(result.counts.suite["go-sdk"], 3); + }); + + it("leaves passes out of the owner facet", () => { + const result = run({ whose: ["suite"] }); + assert.equal(result.total, 2); + assert.equal(result.counts.outcome.pass, undefined); + }); + + it("filters on issue links and on changes since the previous release", () => { + assert.deepEqual(run({ issue: "linked" }).rows.map((r) => r.prepared.test), ["PutObjectTagged"]); + const fixed = run({ changed: "fixed" }); + assert.deepEqual(fixed.rows.map((r) => `${r.prepared.test}`), ["PutObjectTagged"]); + assert.deepEqual(fixed.rows[0].mask, [true, false]); + assert.equal(fixed.counts.changed.fixed, 1); + }); + + it("finds untested operations by their outcome and reason text", () => { + const result = run({ outcomes: ["untested"], q: "destructive" }); + assert.deepEqual(result.untested.map((u) => u.op), ["DeleteLoadBalancer"]); + assert.equal(result.rows.length, 0); + }); +}); + +describe("compat detail and links", () => { + it("groups clients that share an outcome, Overcast's gaps first", () => { + const r = report("v1", { "CreateBucket@go-sdk": "suite-not-written", "CreateBucket@rust-sdk": "not-emulated" }); + const test = r.services[0].groups[0].tests[0]; + const groups = groupResults(test, r.suites, reasonByCode(r)); + assert.deepEqual( + groups.map((g) => [g.reason, g.entries.map((e) => e.suite.id)]), + [ + ["not-emulated", ["rust-sdk"]], + ["suite-not-written", ["go-sdk"]], + ], + ); + const same = report("v1", { "CreateBucket@go-sdk": "not-emulated", "CreateBucket@rust-sdk": "not-emulated" }); + assert.equal(groupResults(same.services[0].groups[0].tests[0], same.suites, reasonByCode(same)).length, 1); + }); + + it("prefills an issue that links itself back through markers", () => { + const suites = [ + { id: "go-sdk", label: "AWS SDK for Go v2" }, + { id: "rust-sdk", label: "AWS SDK for Rust" }, + ]; + const input = { issueRepo: "o/r", service: "s3", serviceName: "S3", group: "s3-crud", test: "PutObjectTagged", op: "PutObject", suites, reasonLabel: "Not emulated yet", error: "x".repeat(5000), version: "v1" }; + assert.deepEqual(issueMarkerTargets(input), ["s3/s3-crud/PutObjectTagged@go-sdk", "s3/s3-crud/PutObjectTagged@rust-sdk"]); + const url = new URL(newIssueUrl(input)); + assert.equal(url.pathname, "/o/r/issues/new"); + assert.equal(url.searchParams.get("labels"), "compat"); + assert.equal(url.searchParams.get("title"), "Compat: S3 PutObject in 2 clients: not emulated yet"); + const body = url.searchParams.get("body") ?? ""; + // The marker scripts/compat-issues.py reads upstream, one target per client. + assert.match(body, //); + assert.ok(url.toString().length < 8000, "the error is trimmed so the URL stays usable"); + }); + + it("writes markdown twins from the same numbers the pages show", () => { + const r = report("v1"); + const options = { origin: "https://overcast.sh", report: r, naming, latest: true }; + const overview = compatOverviewMarkdown(options); + assert.match(overview, /^# Overcast compatibility report, v1/); + assert.match(overview, /Of the 4 results that measure Overcast, 3 pass \(75%\)/); + assert.match(overview, /\[S3\]\(https:\/\/overcast\.sh\/compat\/s3\/index\.md\) \| 66\.6% \| 1\/2 \| 1\/1 \|/); + assert.match(overview, /\[Not emulated yet\]\(https:\/\/overcast\.sh\/compat\/reason\/not-emulated\/index\.md\) \(1\)/); + + const service = compatServiceMarkdown("s3", options); + assert.match(service, /### PutObjectTagged \(PutObject\)/); + assert.match(service, /- AWS SDK for Go v2: Not emulated yet\./); + // A test not yet written is condensed to one line per client, never listed as a failure. + assert.match(service, /- Test not written for this SDK, AWS SDK for Rust: 1 test \(PutObjectTagged\)/); + assert.doesNotMatch(service, /- AWS SDK for Rust: Test not written/); + + const reason = compatReasonMarkdown("untested", options); + assert.match(reason, /- DeleteLoadBalancer: destructive/); + + // An archived release links to the explorer, since it has no per-service pages. + assert.match(compatOverviewMarkdown({ ...options, latest: false }), /\/compat\/explore\/\?release=v1&outcome=not-emulated/); + }); +}); diff --git a/src/lib/compat-report.ts b/src/lib/compat-report.ts new file mode 100644 index 0000000..3862383 --- /dev/null +++ b/src/lib/compat-report.ts @@ -0,0 +1,601 @@ +/** + * compat-report — the public compatibility report, as the site reads it. + * + * Upstream, every Overcast release ships `compat-report.json` (written by + * `go run ./cmd/compat --publish-report`, schema at compat/report.schema.json in + * overcast-sh/overcast). It is one document per release: every compat test, every + * suite's result for it, and for anything that did not pass, exactly one reason + * code that says why and whose move it is. scripts/sync-overcast-content.ts copies + * each release's report into src/generated/compat/. + * + * Everything here is pure (data in, data out) so `npm test` covers it with no build. + * The pages under src/pages/compat/ and the explorer script both work from these + * helpers, which is what keeps the static pages and the filtered view agreeing on + * what a number means. + */ + +export type Whose = "overcast" | "suite" | "sdk" | "environment" | "none"; +export type ResultStatus = "pass" | "fail" | "skip" | "unimplemented" | "na"; + +export interface CompatIssue { + number: number; + url: string; + title?: string; + state?: string; +} + +export interface CompatMismatch { + op: string; + params?: string; + kind: string; + path?: string; + expected: string; + actual: string; + scenario: string; + step: string; +} + +export interface CompatResult { + status: ResultStatus; + reason?: string; + durationMs?: number; + error?: string; + mismatch?: CompatMismatch; + blockedBy?: string[]; + issue?: CompatIssue; +} + +export interface CompatTest { + name: string; + op: string; + depends?: string[]; + requires?: string[]; + results: Record; +} + +export interface CompatGroup { + id: string; + generated?: boolean; + state?: string; + scenario?: string; + tests: CompatTest[]; +} + +export interface CompatGap { + op: string; + group: string; + reason: string; + code: string; + detail: string; + issue?: CompatIssue; +} + +export interface CompatTotals { + cells: number; + pass: number; + byReason: Record; + untested?: number; +} + +export interface CompatService { + id: string; + totals: CompatTotals; + groups: CompatGroup[]; + untested?: CompatGap[]; +} + +export interface CompatSuite { + id: string; + label: string; + kind: "sdk" | "cli" | "iac"; + language: string; + totals: CompatTotals; +} + +export interface CompatReason { + code: string; + label: string; + summary: string; + whose: Whose; +} + +export interface CompatReport { + schema: number; + version: string; + commit?: string; + generatedAt: string; + runStartedAt?: string; + runFinishedAt?: string; + issueRepo: string; + reasons: CompatReason[]; + suites: CompatSuite[]; + totals: CompatTotals; + services: CompatService[]; +} + +/** One entry of src/generated/compat/index.json, newest release first. */ +export interface CompatReportIndexEntry { + tag: string; + publishedAt: string | null; + prerelease: boolean; + /** Where the report came from: a release asset, or a local file in development. */ + source: "release" | "local"; +} + +/** The schema version this site renders. A report with any other is skipped by the sync. */ +export const SUPPORTED_SCHEMA = 1; + +// ── Owners ──────────────────────────────────────────────────────────────────── + +/** The reader-facing heading for each owner, in the order the overview lists them. */ +export const WHOSE_ORDER: readonly Whose[] = ["overcast", "suite", "sdk", "environment", "none"]; + +export const WHOSE_LABELS: Record = { + overcast: { + title: "Overcast gaps", + blurb: "Results Overcast itself has to fix: behaviour that differs from AWS, operations with no emulation yet, and the steps those block.", + }, + suite: { + title: "Test coverage gaps", + blurb: "Tests that have not been written or settled for a language yet. Overcast may well handle these calls; nothing has checked.", + }, + sdk: { + title: "SDK limits", + blurb: "Operations an SDK or the CLI has no API for, so there is nothing to call.", + }, + environment: { + title: "Environment", + blurb: "Tests that need Docker, SMTP or network access the release run does not provide.", + }, + none: { + title: "Not tested", + blurb: "Operations with no test at all. The generator declined to write one and recorded why.", + }, +}; + +export type Tone = "success" | "warning" | "danger" | "muted" | "accent"; + +/** Chip tone per owner: Overcast's gaps are the ones that bear on "will my call work". */ +export const WHOSE_TONE: Record = { + overcast: "danger", + suite: "muted", + sdk: "muted", + environment: "muted", + none: "warning", +}; + +/** Per-reason tone, finer than the owner's: a 501 is a known gap, a mismatch is a bug. */ +export function reasonTone(code: string | undefined, reasons: readonly CompatReason[]): Tone { + if (!code) return "success"; + if (code === "behaviour-mismatch") return "danger"; + if (code === "not-emulated" || code === "quarantined-flaky" || code === "dependency-failed") return "warning"; + const reason = reasons.find((r) => r.code === code); + return reason?.whose === "overcast" ? "warning" : "muted"; +} + +export function reasonByCode(report: Pick): Map { + return new Map(report.reasons.map((reason) => [reason.code, reason])); +} + +/** The owner of a reason code. Unknown codes are treated as Overcast's, the cautious reading. */ +export function whoseOf(code: string, reasons: ReadonlyMap): Whose { + return reasons.get(code)?.whose ?? "overcast"; +} + +// ── Services ────────────────────────────────────────────────────────────────── + +/** + * Compat service ids that are spelt differently from the support matrix's (and so from the + * service catalog and the docs). Everything else matches 1:1. + */ +const SUPPORT_ALIASES: Record = { + "elastic-load-balancing": "elbv2", +}; + +/** Names for compat services the support matrix has no row for. */ +const EXTRA_NAMES: Record = { + cdk: "CDK stacks", + servicediscovery: "Cloud Map", + batch: "Batch", + "elastic-load-balancing": "ELBv2", +}; + +/** The support matrix's service id for a compat service id. */ +export function supportServiceId(compatId: string): string { + return SUPPORT_ALIASES[compatId] ?? compatId; +} + +export interface ServiceNaming { + /** Display names by support-matrix service id, from service-support.json. */ + displayNames: ReadonlyMap; + /** Doc slugs by support-matrix service id. A service with no doc has no entry. */ + docSlugs: ReadonlyMap; +} + +export function serviceDisplayName(compatId: string, naming: ServiceNaming): string { + return naming.displayNames.get(supportServiceId(compatId)) ?? EXTRA_NAMES[compatId] ?? compatId; +} + +export function serviceDocPath(compatId: string, naming: ServiceNaming): string | null { + const slug = naming.docSlugs.get(supportServiceId(compatId)); + return slug ? `/docs/services/${slug}/` : null; +} + +/** A URL-safe anchor for one test row, stable across releases. */ +export function testAnchor(group: string, test: string): string { + return `${group}--${test}`.toLowerCase().replace(/[^a-z0-9-]+/g, "-"); +} + +// ── Numbers ─────────────────────────────────────────────────────────────────── + +/** + * The cells that measure Overcast: passes plus Overcast's own gaps. A test nobody has + * written, or one an SDK cannot express, says nothing about the emulator, so it stays out + * of the rate. A 501 does count against it, unlike in the suites' own pass rate: "will this + * call work against Overcast" is the question a reader brings here, and for a 501 it won't. + */ +export function measured(totals: CompatTotals, reasons: ReadonlyMap): { pass: number; ran: number } { + let gaps = 0; + for (const [code, count] of Object.entries(totals.byReason)) { + if (countsAgainst(code, reasons)) gaps += count; + } + return { pass: totals.pass, ran: totals.pass + gaps }; +} + +/** + * Reasons that are Overcast's but have no result of their own. A test blocked by an earlier + * step never ran: counting it would charge the one failure that blocked it once per dependent + * test. It is listed with Overcast's gaps, and left out of the rate and the headline count. + */ +export const UNMEASURED_REASONS: ReadonlySet = new Set(["dependency-failed"]); + +/** Whether a reason counts against Overcast's pass rate. */ +export function countsAgainst(code: string, reasons: ReadonlyMap): boolean { + return whoseOf(code, reasons) === "overcast" && !UNMEASURED_REASONS.has(code); +} + +export function percent(part: number, whole: number): number { + return whole ? Math.floor((part / whole) * 1000) / 10 : 0; +} + +/** Totals for one owner: every reason it owns, summed. */ +export function countByWhose(totals: CompatTotals, reasons: ReadonlyMap): Record { + const out: Record = { overcast: 0, suite: 0, sdk: 0, environment: 0, none: totals.untested ?? 0 }; + for (const [code, count] of Object.entries(totals.byReason)) out[whoseOf(code, reasons)] += count; + return out; +} + +export interface MatrixCell { + suite: string; + pass: number; + ran: number; + /** Every cell for this service and suite, whatever its reason. */ + cells: number; + /** Counts by reason code, for the cell's tooltip and its screen-reader text. */ + byReason: Record; +} + +/** One service's row of the service × suite matrix. */ +export function matrixRow(service: CompatService, suites: readonly CompatSuite[], reasons: ReadonlyMap): MatrixCell[] { + return suites.map((suite) => { + const cell: MatrixCell = { suite: suite.id, pass: 0, ran: 0, cells: 0, byReason: {} }; + for (const group of service.groups) { + for (const test of group.tests) { + const result = test.results[suite.id]; + if (!result) continue; + cell.cells += 1; + if (!result.reason) { + cell.pass += 1; + cell.ran += 1; + continue; + } + cell.byReason[result.reason] = (cell.byReason[result.reason] ?? 0) + 1; + if (countsAgainst(result.reason, reasons)) cell.ran += 1; + } + } + return cell; + }); +} + +/** A service's pass/ran across the given suites: the sum of its matrix row. */ +export function serviceMeasured(service: CompatService, suites: readonly CompatSuite[], reasons: ReadonlyMap): { pass: number; ran: number } { + return matrixRow(service, suites, reasons).reduce((sum, cell) => ({ pass: sum.pass + cell.pass, ran: sum.ran + cell.ran }), { pass: 0, ran: 0 }); +} + +/** Tone for a pass rate: all green, mostly there, or a real gap. No results at all is muted. */ +export function rateTone(pass: number, ran: number): Tone { + if (ran === 0) return "muted"; + if (pass === ran) return "success"; + return pass / ran >= 0.8 ? "warning" : "danger"; +} + +// ── Operations ──────────────────────────────────────────────────────────────── + +export interface OperationSummary { + service: string; + op: string; + /** At least one suite passed a test that calls it. */ + verified: boolean; + /** At least one Overcast-owned gap on a test that calls it. */ + failing: boolean; +} + +/** Per-operation rollup, for the headline "operations verified" number. */ +export function operationSummaries(report: CompatReport): OperationSummary[] { + const reasons = reasonByCode(report); + const byKey = new Map(); + for (const service of report.services) { + for (const group of service.groups) { + for (const test of group.tests) { + const key = `${service.id}/${test.op}`; + const summary = byKey.get(key) ?? { service: service.id, op: test.op, verified: false, failing: false }; + for (const result of Object.values(test.results)) { + if (!result.reason) summary.verified = true; + else if (countsAgainst(result.reason, reasons)) summary.failing = true; + } + byKey.set(key, summary); + } + } + } + return [...byKey.values()]; +} + +// ── Release-over-release ───────────────────────────────────────────────────── + +export interface CellChange { + service: string; + group: string; + test: string; + op: string; + suite: string; + /** Reason code before, "" for a pass, null when the cell did not exist. */ + from: string | null; + to: string | null; +} + +function cellMap(report: CompatReport): Map { + const out = new Map(); + for (const service of report.services) { + for (const group of service.groups) { + for (const test of group.tests) { + for (const [suite, result] of Object.entries(test.results)) { + out.set(`${service.id}/${group.id}/${test.name}@${suite}`, { service: service.id, group: group.id, test, suite, reason: result.reason ?? "" }); + } + } + } + } + return out; +} + +/** + * Every cell whose outcome moved between two releases: newly passing, newly failing, or + * changed from one reason to another. Cells that appear or disappear (a test added or + * retired) are included with a null side, so a caller can decide whether they count. + */ +export function diffReports(previous: CompatReport, current: CompatReport): CellChange[] { + const before = cellMap(previous); + const after = cellMap(current); + const changes: CellChange[] = []; + for (const [key, cell] of after) { + const old = before.get(key); + const from = old ? old.reason : null; + if (from === cell.reason) continue; + changes.push({ service: cell.service, group: cell.group, test: cell.test.name, op: cell.test.op, suite: cell.suite, from, to: cell.reason }); + } + for (const [key, cell] of before) { + if (after.has(key)) continue; + changes.push({ service: cell.service, group: cell.group, test: cell.test.name, op: cell.test.op, suite: cell.suite, from: cell.reason, to: null }); + } + return changes; +} + +/** A change that made a previously-running cell pass. */ +export const isFixed = (change: CellChange): boolean => change.to === "" && change.from !== null && change.from !== ""; +/** A change that turned a passing cell into anything else. */ +export const isRegressed = (change: CellChange): boolean => change.from === "" && change.to !== null && change.to !== ""; + +// ── The explorer's index ───────────────────────────────────────────────────── + +/** + * The compact form the explorer fetches: one row per test, one slot per suite. Strings that + * repeat (services, groups, reasons, suites) are dictionary-encoded, so ~1,200 tests across + * eight suites come to a small fraction of the full report. + * + * A cell is a reason index, -1 for a pass, or null when the suite does not run the test. + * Errors are trimmed: the explorer shows enough to recognise the failure and links to the + * service page for the whole message. + */ +export interface ExplorerIndex { + version: string; + /** owner/repo that issue numbers refer to. */ + issueRepo: string; + suites: { id: string; label: string }[]; + reasons: CompatReason[]; + services: { id: string; name: string }[]; + groups: string[]; + /** [serviceIdx, groupIdx, test, op ("" when equal to test), cells, details] */ + rows: ExplorerRow[]; + /** [serviceIdx, op, gap code, detail, issue number or 0] */ + untested: [number, string, string, string, number][]; + /** "service/group/test@suite" of every cell whose outcome moved since the previous release. */ + changed?: { fixed: string[]; regressed: string[]; previous: string }; +} + +export type ExplorerCell = number | null; +/** Per-suite detail, keyed by suite index: [error snippet, issue number or 0, blocked by]. */ +export type ExplorerDetail = Record; +export type ExplorerRow = [number, number, string, string, ExplorerCell[], ExplorerDetail]; + +export const EXPLORER_ERROR_LIMIT = 240; + +export function trimError(error: string | undefined, limit = EXPLORER_ERROR_LIMIT): string { + if (!error) return ""; + const flat = error.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit - 1)}…` : flat; +} + +export function buildExplorerIndex(report: CompatReport, naming: ServiceNaming, previous?: CompatReport): ExplorerIndex { + const reasonIndex = new Map(report.reasons.map((reason, i) => [reason.code, i])); + const suiteIndex = new Map(report.suites.map((suite, i) => [suite.id, i])); + const groups: string[] = []; + const groupIndex = new Map(); + const rows: ExplorerRow[] = []; + const untested: ExplorerIndex["untested"] = []; + + report.services.forEach((service, serviceIdx) => { + for (const group of service.groups) { + let g = groupIndex.get(group.id); + if (g === undefined) { + g = groups.push(group.id) - 1; + groupIndex.set(group.id, g); + } + for (const test of group.tests) { + const cells: ExplorerCell[] = report.suites.map(() => null); + const details: ExplorerDetail = {}; + for (const [suiteId, result] of Object.entries(test.results)) { + const s = suiteIndex.get(suiteId); + if (s === undefined) continue; + if (!result.reason) { + cells[s] = -1; + continue; + } + // An unknown code would index nothing; it is appended so it still renders. + let r = reasonIndex.get(result.reason); + if (r === undefined) { + r = reasonIndex.size; + reasonIndex.set(result.reason, r); + } + cells[s] = r; + const error = trimError(result.error); + const issue = result.issue?.number ?? 0; + const blocked = result.blockedBy?.join(", ") ?? ""; + if (error || issue || blocked) details[s] = [error, issue, blocked]; + } + rows.push([serviceIdx, g, test.name, test.op === test.name ? "" : test.op, cells, details]); + } + } + for (const gap of service.untested ?? []) { + untested.push([serviceIdx, gap.op, gap.code, gap.detail, gap.issue?.number ?? 0]); + } + }); + + const reasons = [...report.reasons]; + for (const [code, i] of reasonIndex) { + if (i >= report.reasons.length) reasons[i] = { code, label: code, summary: "", whose: "overcast" }; + } + + const index: ExplorerIndex = { + version: report.version, + issueRepo: report.issueRepo, + suites: report.suites.map((suite) => ({ id: suite.id, label: suite.label })), + reasons, + services: report.services.map((service) => ({ id: service.id, name: serviceDisplayName(service.id, naming) })), + groups, + rows, + untested, + }; + if (previous) { + const changes = diffReports(previous, report); + const key = (c: CellChange) => `${c.service}/${c.group}/${c.test}@${c.suite}`; + index.changed = { + previous: previous.version, + fixed: changes.filter(isFixed).map(key), + regressed: changes.filter(isRegressed).map(key), + }; + } + return index; +} + +// ── Presentation ───────────────────────────────────────────────────────────── + +/** Column headings for the suites: a matrix of eight full SDK names does not fit a phone. */ +export const SUITE_SHORT: Record = { + "node-js-sdk": "JS", + "python-sdk": "Python", + "go-sdk": "Go", + cli: "CLI", + cdk: "CDK", + "java-sdk": "Java", + "dotnet-sdk": ".NET", + "rust-sdk": "Rust", +}; + +export const suiteShort = (id: string): string => SUITE_SHORT[id] ?? id; + +/** The word in a matrix cell. The full label and summary come from the report itself. */ +export const REASON_SHORT: Record = { + "behaviour-mismatch": "Differs", + "not-emulated": "501", + "quarantined-flaky": "Flaky", + "dependency-failed": "Blocked", + "suite-not-written": "No test", + candidate: "Soaking", + "not-reported": "Missing", + "sdk-lacks-api": "N/A", + "needs-environment": "Env", + "other-skip": "Skipped", + untested: "Untested", +}; + +export const reasonShort = (code: string | undefined): string => (code ? (REASON_SHORT[code] ?? code) : "Pass"); + +/** Symbols in CompatIconSprite, one per reason that has its own glyph. */ +export const COMPAT_ICON_KEYS = [ + "pass", + "behaviour-mismatch", + "not-emulated", + "quarantined-flaky", + "dependency-failed", + "suite-not-written", + "candidate", + "sdk-lacks-api", + "needs-environment", + "other", +] as const; + +/** The sprite symbol for a result: its reason's own glyph, or a neutral one. */ +export function compatIconKey(code: string | undefined): (typeof COMPAT_ICON_KEYS)[number] { + if (!code) return "pass"; + return (COMPAT_ICON_KEYS as readonly string[]).includes(code) ? (code as (typeof COMPAT_ICON_KEYS)[number]) : "other"; +} + +/** Every reason that has at least one cell, in the report's own order. */ +export function reasonsInUse(report: CompatReport): CompatReason[] { + return report.reasons.filter((reason) => + reason.code === "untested" ? (report.totals.untested ?? 0) > 0 : (report.totals.byReason[reason.code] ?? 0) > 0, + ); +} + +/** The count for one reason, untested included. */ +export function reasonCount(report: CompatReport, code: string): number { + return code === "untested" ? (report.totals.untested ?? 0) : (report.totals.byReason[code] ?? 0); +} + +/** A short commit for display. */ +export const shortCommit = (sha: string | undefined): string => (sha ? sha.slice(0, 9) : ""); + +export interface ResultGroup { + reason: string; + entries: { suite: { id: string; label: string }; result: CompatResult }[]; +} + +/** + * A test's non-passing results, grouped so that clients sharing an outcome read once: same + * reason, same parsed evidence, same tracking issue. Messages may still differ (every SDK words + * a 501 its own way); the detail view lists them together. Groups keep suite order, and + * Overcast's own gaps come first. + */ +export function groupResults(test: CompatTest, suites: readonly { id: string; label: string }[], reasons: ReadonlyMap): ResultGroup[] { + const groups = new Map(); + for (const suite of suites) { + const result = test.results[suite.id]; + if (!result?.reason) continue; + const m = result.mismatch; + const key = [result.reason, result.issue?.number ?? 0, m ? `${m.kind}|${m.path}|${m.expected}|${m.actual}` : "", (result.blockedBy ?? []).join(",")].join("|"); + const group = groups.get(key) ?? { reason: result.reason, entries: [] }; + group.entries.push({ suite: { id: suite.id, label: suite.label }, result }); + groups.set(key, group); + } + const rank = (code: string) => (whoseOf(code, reasons) === "overcast" ? 0 : 1); + return [...groups.values()].sort((a, b) => rank(a.reason) - rank(b.reason)); +} diff --git a/src/lib/site-markdown.ts b/src/lib/site-markdown.ts index 915c6cb..3dc1b77 100644 --- a/src/lib/site-markdown.ts +++ b/src/lib/site-markdown.ts @@ -18,6 +18,22 @@ * build and no collection. */ import { coverageTierLabel, type ServiceSupportManifest } from "./generated-content.ts"; +import { + WHOSE_LABELS, + WHOSE_ORDER, + countByWhose, + matrixRow, + measured, + percent, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + serviceMeasured, + type CompatReason, + type CompatReport, + type ServiceNaming, +} from "./compat-report.ts"; export interface MarkdownPageOptions { /** Absolute origin, so a reader that fetched this file can follow every link in it. */ @@ -197,3 +213,178 @@ export function docsIndexMarkdown({ sections, origin }: DocsIndexOptions): strin ...blocks, ]); } + +// ── Compatibility report ───────────────────────────────────────────────────── + +export interface CompatMarkdownOptions extends MarkdownPageOptions { + report: CompatReport; + naming: ServiceNaming; + /** The newest release has per-service pages; an archived one links to the explorer. */ + latest: boolean; +} + +/** "1 result is" / "3 results are". */ +function plural(n: number, one: string, many: string): string { + return `${n} ${n === 1 ? one : many}`; +} + +/** One reason as a markdown line, for the "why" sections. */ +function reasonLine(report: CompatReport, reason: CompatReason, href: string): string { + return `- [${reason.label}](${href}) (${reasonCount(report, reason.code)}): ${reason.summary}`; +} + +/** + * /compat/index.md (and /compat/history//index.md) — the overview: the headline numbers, + * every reason by owner, and the service × client matrix as a table. + */ +export function compatOverviewMarkdown({ report, naming, latest, origin }: CompatMarkdownOptions): string { + const reasons = reasonByCode(report); + const headline = measured(report.totals, reasons); + const whose = countByWhose(report.totals, reasons); + const inUse = reasonsInUse(report); + const reasonHref = (code: string) => + latest + ? absolute(origin, `/compat/reason/${code}/index.md`) + : absolute(origin, `/compat/explore/?release=${encodeURIComponent(report.version)}&outcome=${code}`); + + const whyBlocks = WHOSE_ORDER.map((owner) => { + const group = inUse.filter((reason) => reason.whose === owner); + if (!group.length) return ""; + return [ + `### ${WHOSE_LABELS[owner].title} (${whose[owner]})`, + "", + WHOSE_LABELS[owner].blurb, + "", + ...group.map((reason) => reasonLine(report, reason, reasonHref(reason.code))), + ].join("\n"); + }).filter(Boolean); + + const services = [...report.services].sort((a, b) => + serviceDisplayName(a.id, naming).localeCompare(serviceDisplayName(b.id, naming)), + ); + const rows = services.map((service) => { + const m = serviceMeasured(service, report.suites, reasons); + const name = serviceDisplayName(service.id, naming); + const cells = matrixRow(service, report.suites, reasons).map((cell) => + cell.cells === 0 ? "—" : cell.ran ? `${cell.pass}/${cell.ran}` : "no test", + ); + const label = latest ? `[${name}](${absolute(origin, `/compat/${service.id}/index.md`)})` : name; + return [label, `${percent(m.pass, m.ran)}%`, ...cells]; + }); + const releaseUrl = `https://github.com/${report.issueRepo}/releases/tag/${report.version}`; + + return page( + `Overcast compatibility report, ${report.version}`, + "> Tested compatibility across six AWS SDKs, the AWS CLI and the CDK, with the reason for every result that does not pass.", + [ + `Of the ${headline.ran} results that measure Overcast, ${headline.pass} pass (${percent(headline.pass, headline.ran)}%). ${plural(headline.ran - headline.pass, "result is", "results are")} for Overcast to fix, ${plural(whose.suite, "is a test", "are tests")} not yet written or settled for a client, and ${plural(whose.none, "operation has", "operations have")} no test. A test not yet written for a language, an operation an SDK cannot call, or a test blocked by an earlier failure is left out of the rate.`, + `The full data is the \`compat-report.json\` asset on the release, ${releaseUrl}. Every result can be searched at ${absolute(origin, "/compat/explore/")}.`, + "## Why results do not pass", + ...whyBlocks, + "## By service and client", + "Passing results over results that measure Overcast.", + table(["Service", "Rate", ...report.suites.map((suite) => suite.label)], rows, "_No results in this report._"), + ], + ); +} + +/** + * /compat//index.md — every test Overcast does not yet pass, with the reason and the + * evidence, per client. Results that say nothing about Overcast (a test not yet written for a + * language, an SDK with no API) are condensed to one line per reason and client, or a service + * with a young Rust suite would bury its three real failures under sixty identical lines. + */ +export function compatServiceMarkdown(serviceId: string, { report, naming, origin }: CompatMarkdownOptions): string { + const service = report.services.find((s) => s.id === serviceId); + const name = serviceDisplayName(serviceId, naming); + if (!service) return page(`${name} compatibility`, "> No results for this service in this report.", []); + const reasons = reasonByCode(report); + const m = serviceMeasured(service, report.suites, reasons); + const labels = new Map(report.suites.map((suite) => [suite.id, suite.label])); + const tests = service.groups.flatMap((group) => group.tests.map((test) => ({ group: group.id, test }))); + const detailed = (code: string) => reasons.get(code)?.whose === "overcast" || code === "candidate"; + + const blocks = tests + .filter(({ test }) => Object.values(test.results).some((result) => result.reason && detailed(result.reason))) + .map(({ group, test }) => { + const lines = Object.entries(test.results) + .filter(([, result]) => result.reason && detailed(result.reason)) + .map(([suite, result]) => { + const reason = reasons.get(result.reason ?? ""); + const issue = result.issue ? ` Tracked in ${result.issue.url}.` : ""; + let evidence = ""; + if (result.mismatch) { + const at = result.mismatch.path ? ` at \`${result.mismatch.path}\`` : ""; + evidence = ` ${result.mismatch.kind}${at}: expected ${result.mismatch.expected}, actual ${result.mismatch.actual}.`; + } else if (result.blockedBy?.length) { + evidence = ` Blocked by ${result.blockedBy.join(", ")}.`; + } + return `- ${labels.get(suite) ?? suite}: ${reason?.label ?? result.reason}.${evidence}${issue}`; + }); + const heading = `### ${test.name}${test.op !== test.name ? ` (${test.op})` : ""}`; + return [heading, "", `Group \`${group}\`.`, "", ...lines].join("\n"); + }); + + // Everything else, one line per reason and client: "AWS SDK for Rust: 60 tests (CreateUser, …)". + const condensed = new Map(); + for (const { test } of tests) { + for (const [suite, result] of Object.entries(test.results)) { + if (!result.reason || detailed(result.reason)) continue; + const key = `${reasons.get(result.reason)?.label ?? result.reason}|${labels.get(suite) ?? suite}`; + condensed.set(key, [...(condensed.get(key) ?? []), test.name]); + } + } + const otherLines = [...condensed].map(([key, names]) => { + const [label, client] = key.split("|"); + return `- ${label}, ${client}: ${names.length} ${names.length === 1 ? "test" : "tests"} (${names.join(", ")})`; + }); + + const untested = (service.untested ?? []).map((gap) => `- ${gap.op} (\`${gap.code}\`): ${gap.detail}`); + + return page(`${name} compatibility, ${report.version}`, `> Compatibility test results for ${name} in Overcast ${report.version}.`, [ + `${tests.length} tests. Of the ${m.ran} results that measure Overcast, ${m.pass} pass (${percent(m.pass, m.ran)}%). ${blocks.length} tests have a result Overcast has to act on, listed below with the evidence.`, + `The overview, with every service, is at ${absolute(origin, "/compat/index.md")}.`, + ...(blocks.length ? ["## Tests Overcast does not yet pass", ...blocks] : []), + ...(otherLines.length ? ["## Not measured", "Results that say nothing about Overcast itself.", otherLines.join("\n")] : []), + ...(untested.length + ? ["## Not tested", "Operations with no test. The generator declined to write one, for the reason given.", untested.join("\n")] + : []), + ]); +} + +/** /compat/reason//index.md — every result with one reason, by service. */ +export function compatReasonMarkdown(code: string, { report, naming, origin }: CompatMarkdownOptions): string { + const reason = report.reasons.find((r) => r.code === code); + const label = reason?.label ?? code; + const labels = new Map(report.suites.map((suite) => [suite.id, suite.label])); + const blocks = report.services + .map((service) => { + const lines = service.groups.flatMap((group) => + group.tests + .map((test) => ({ + test, + suites: Object.entries(test.results) + .filter(([, result]) => result.reason === code) + .map(([suite]) => labels.get(suite) ?? suite), + })) + .filter(({ suites }) => suites.length) + .map(({ test, suites }) => `- ${test.name}${test.op !== test.name ? ` (${test.op})` : ""}: ${suites.join(", ")}`), + ); + const gaps = code === "untested" ? (service.untested ?? []).map((gap) => `- ${gap.op}: ${gap.detail}`) : []; + const all = [...lines, ...gaps]; + return all.length ? [`## ${serviceDisplayName(service.id, naming)}`, "", ...all].join("\n") : ""; + }) + .filter(Boolean); + + return page(`${label}: Overcast ${report.version} compatibility`, `> ${reason?.summary ?? label}`, [ + `${reasonCount(report, code)} ${code === "untested" ? "operations" : "results"}. The overview is at ${absolute(origin, "/compat/index.md")}.`, + ...blocks, + ]); +} + +/** /compat/explore/index.md — the explorer is interactive; its twin says where the data is. */ +export function compatExploreMarkdown({ origin }: MarkdownPageOptions): string { + return page("Explore compatibility results", "> Search and filter every Overcast compatibility test result.", [ + `The page filters one compact index per release, served at ${absolute(origin, "/compat/data/")}.json. The full report for a release is its \`compat-report.json\` release asset, and the overview is at ${absolute(origin, "/compat/index.md")}.`, + ]); +} diff --git a/src/lib/site-pages.ts b/src/lib/site-pages.ts index 6c6cea3..d3f75e3 100644 --- a/src/lib/site-pages.ts +++ b/src/lib/site-pages.ts @@ -36,6 +36,17 @@ export const SITE_PAGES: readonly SitePage[] = [ title: "Support matrix", description: "Overcast service and operation implementation status.", }, + { + path: "/compat/", + title: "Compatibility report", + description: + "Overcast's compatibility test results across six AWS SDKs, the AWS CLI and the CDK, with the reason for every result that does not pass.", + }, + { + path: "/compat/explore/", + title: "Explore compatibility results", + description: "Search and filter every Overcast compatibility test result by operation, client, reason and tracking issue.", + }, { path: "/downloads/", title: "Downloads", diff --git a/src/pages/[...slug].md.ts b/src/pages/[...slug].md.ts index 8fd7aaf..e0371f3 100644 --- a/src/pages/[...slug].md.ts +++ b/src/pages/[...slug].md.ts @@ -24,7 +24,13 @@ import { getServiceSupport } from "../lib/generated-content"; import { repositoryUrl, websiteGitHubRepo } from "../lib/github-links"; import { newestReleaseWithAssets, sortReleasesNewestFirst } from "../lib/releases"; import { LLMS_SECTION_ORDER } from "../lib/llms-txt"; +import { getCompatIndex, getCompatReport, getServiceNaming } from "../lib/compat-data"; +import { reasonsInUse } from "../lib/compat-report"; import { + compatExploreMarkdown, + compatOverviewMarkdown, + compatReasonMarkdown, + compatServiceMarkdown, consoleMarkdown, contributingMarkdown, docsIndexMarkdown, @@ -144,7 +150,34 @@ export const getStaticPaths: GetStaticPaths = async () => { }, ]; - return [...docPaths, ...sitePaths]; + // The compatibility report: the newest release at /compat/ with its service and reason + // pages, and each older one at /compat/history// — the same set the .astro routes build. + const compatIndex = await getCompatIndex(); + const naming = await getServiceNaming(); + const compatPaths: typeof sitePaths = []; + for (const [i, entry] of compatIndex.entries()) { + const report = await getCompatReport(entry.tag); + if (!report) continue; + const compat = { ...options, report, naming, latest: i === 0 }; + if (i > 0) { + compatPaths.push({ params: { slug: `compat/history/${entry.tag}/index` }, props: { body: compatOverviewMarkdown(compat) } }); + continue; + } + compatPaths.push( + { params: { slug: "compat/index" }, props: { body: compatOverviewMarkdown(compat) } }, + { params: { slug: "compat/explore/index" }, props: { body: compatExploreMarkdown(options) } }, + ...report.services.map((service) => ({ + params: { slug: `compat/${service.id}/index` }, + props: { body: compatServiceMarkdown(service.id, compat) }, + })), + ...reasonsInUse(report).map((reason) => ({ + params: { slug: `compat/reason/${reason.code}/index` }, + props: { body: compatReasonMarkdown(reason.code, compat) }, + })), + ); + } + + return [...docPaths, ...sitePaths, ...compatPaths]; }; export const GET: APIRoute = ({ props }) => { diff --git a/src/pages/compat/[service].astro b/src/pages/compat/[service].astro new file mode 100644 index 0000000..4e68d78 --- /dev/null +++ b/src/pages/compat/[service].astro @@ -0,0 +1,433 @@ +--- +// One service's compatibility results for the newest release: every test, every client, and +// for each result that does not pass, the reason and the evidence. Complete without script; +// the filter bar only hides rows. Anchors (#--) are stable across releases, so a +// link to a test keeps working after the next one ships. +import type { GetStaticPaths } from "astro"; +import { ArrowLeft, BookOpen, ListFilter, Search, ShieldCheck, X } from "@lucide/astro"; +import SiteLayout from "../../components/SiteLayout.astro"; +import ServiceIcon from "../../components/ServiceIcon.astro"; +import Chip from "../../components/Chip.astro"; +import CompatCell from "../../components/CompatCell.astro"; +import CompatIconSprite from "../../components/CompatIconSprite.astro"; +import CompatResultDetail from "../../components/CompatResultDetail.astro"; +import { getLatestCompatRelease, getServiceNaming } from "../../lib/compat-data"; +import { + matrixRow, + serviceMeasured, + percent, + rateTone, + reasonByCode, + serviceDisplayName, + serviceDocPath, + suiteShort, + supportServiceId, + groupResults, + testAnchor, + trimError, + whoseOf, +} from "../../lib/compat-report"; + +export const getStaticPaths: GetStaticPaths = async () => { + const release = await getLatestCompatRelease(); + return (release?.report.services ?? []).map((service) => ({ params: { service: service.id } })); +}; + +const { service: serviceId } = Astro.params as { service: string }; +const [release, naming] = await Promise.all([getLatestCompatRelease(), getServiceNaming()]); +const report = release!.report; +const service = report.services.find((s) => s.id === serviceId)!; +const reasons = reasonByCode(report); +const name = serviceDisplayName(service.id, naming); +const docPath = serviceDocPath(service.id, naming); +const suites = report.suites.filter((suite) => service.groups.some((g) => g.tests.some((t) => t.results[suite.id]))); +const cells = matrixRow(service, suites, reasons); +const headline = serviceMeasured(service, suites, reasons); +const testCount = service.groups.reduce((sum, group) => sum + group.tests.length, 0); +const toneVar = { success: "--oc-success", warning: "--oc-warning", danger: "--oc-danger", muted: "--oc-muted", accent: "--oc-accent" } as const; + +// What the filter script needs to know about each reason: only whether it is Overcast's. +const overcastCodes = report.reasons.filter((reason) => reason.whose === "overcast").map((reason) => reason.code); +const showOptions = [ + { value: "all", label: "All results" }, + { value: "failing", label: "Not passing" }, + { value: "overcast", label: "Overcast gaps" }, +]; +--- + + + +
+ + Compatibility report {report.version} + +

+ + {name} +

+

+ {testCount} tests across {suites.length} {suites.length === 1 ? "client" : "clients"}. Of the {headline.ran} results that measure + Overcast, {headline.pass} pass ({percent(headline.pass, headline.ran)}%). +

+
+ { + docPath && ( + + + {name} reference + + ) + } + + Support matrix + + + Explore with filters + +
+ +
    + { + cells.map((cell, i) => ( +
  • + + {suiteShort(suites[i].id)} + + + {cell.ran ? ( + <> + {cell.pass} + /{cell.ran} + + ) : ( + no test + )} + passing in {suites[i].label} + +
    • + )) + } +
+ +
+
+ + + + +
+
+ { + showOptions.map((option) => ( + + )) + } +
+ +
+

+ {testCount} tests +

+ + { + service.groups.map((group) => ( +
+

+ + {group.id} + + {group.generated && generated from the AWS model} + {group.state === "candidate" && soaking} +

+
+ + + + + {suites.map((suite) => ( + + ))} + + + + {group.tests.map((test) => { + const anchor = testAnchor(group.id, test.name); + const failing = suites.filter((suite) => test.results[suite.id]?.reason); + const cellsAttr = suites.map((suite) => `${suite.id}=${test.results[suite.id] ? (test.results[suite.id].reason ?? "pass") : ""}`).join(";"); + const errors = Object.values(test.results) + .map((r) => [trimError(r.error), r.issue ? `#${r.issue.number}` : ""].join(" ")) + .join(" "); + const overcastFailing = failing.filter((suite) => whoseOf(test.results[suite.id].reason!, reasons) === "overcast").length; + return ( + <> + + + {suites.map((suite) => ( + + ))} + + {failing.length > 0 && ( + + + + )} + + ); + })} + +
Test + + {suiteShort(suite.id)} + +
+ + {test.name} + + {test.op !== test.name && {test.op}} + {test.requires?.length ? needs {test.requires.join(", ")} : null} + + +
+
0 ? "" : undefined}> + + Why {failing.length === 1 ? "1 result does" : `${failing.length} results do`} not pass + {overcastFailing > 0 && · {overcastFailing} on Overcast} + +
+ {groupResults(test, suites, reasons).map((shared) => ( +
entry.suite.id).join(" ")}> + +
+ ))} +
+
+
+
+
+ )) + } + + + + { + service.untested?.length ? ( +
+

+ Not tested ({service.untested.length}) +

+

+ Operations with no test at all. The test generator declined to write one, for the reason given. +

+
+ + + + + + + + + + {service.untested.map((gap) => ( + + + + + + ))} + +
OperationReasonDetail
+ {gap.op} + + {gap.code} + {gap.issue && ( + + #{gap.issue.number} + + )} + {gap.detail}
+
+
+ ) : null + } +
+ + + diff --git a/src/pages/compat/data/[tag].json.ts b/src/pages/compat/data/[tag].json.ts new file mode 100644 index 0000000..596b85f --- /dev/null +++ b/src/pages/compat/data/[tag].json.ts @@ -0,0 +1,18 @@ +import type { APIRoute, GetStaticPaths } from "astro"; +import { getCompatIndex, getCompatRelease, getServiceNaming } from "../../../lib/compat-data"; +import { buildExplorerIndex } from "../../../lib/compat-report"; + +// https://overcast.sh/compat/data/.json — the explorer's compact index for one release +// (see ExplorerIndex in src/lib/compat-report.ts). Fetched only by /compat/explore/, so the +// static pages never pay for it. The full report is the release asset itself. +export const getStaticPaths: GetStaticPaths = async () => { + const index = await getCompatIndex(); + return index.map((entry) => ({ params: { tag: entry.tag } })); +}; + +export const GET: APIRoute = async ({ params }) => { + const release = await getCompatRelease(params.tag as string); + if (!release) return new Response("not found", { status: 404 }); + const body = buildExplorerIndex(release.report, await getServiceNaming(), release.previous ?? undefined); + return new Response(JSON.stringify(body), { headers: { "Content-Type": "application/json; charset=utf-8" } }); +}; diff --git a/src/pages/compat/explore.astro b/src/pages/compat/explore.astro new file mode 100644 index 0000000..25161eb --- /dev/null +++ b/src/pages/compat/explore.astro @@ -0,0 +1,424 @@ +--- +// Every compatibility result for a release, filtered as you type. The data is one compact +// index per release (/compat/data/.json, see ExplorerIndex); what matches is decided by +// src/lib/compat-filter.ts, which is unit-tested, and this page only renders it. Every view is +// a URL: ?release=&q=&outcome=&whose=&suite=&issue=&changed=. +import { ArrowLeft, ListFilter, Search, X } from "@lucide/astro"; +import SiteLayout from "../../components/SiteLayout.astro"; +import { getCompatIndex } from "../../lib/compat-data"; + +const index = await getCompatIndex(); +const releases = index.map((entry) => entry.tag); +--- + + +
+ + Compatibility report + +

Explore every result

+

+ Search by operation, test, service, error message or issue number, then narrow by outcome, client and owner. Every + view is a link: copy the address to share exactly what you are looking at. +

+ + { + releases.length === 0 ? ( +

No compatibility report has been published yet.

+ ) : ( + <> +
+
+ + + + +
+ +
+ +
+ +
+
+

+ Loading results… +

+ +
+
+ +
+
+ + ) + } +
+ + + diff --git a/src/pages/compat/history/[tag].astro b/src/pages/compat/history/[tag].astro new file mode 100644 index 0000000..31bd56c --- /dev/null +++ b/src/pages/compat/history/[tag].astro @@ -0,0 +1,25 @@ +--- +// An earlier release's report. The newest release lives at /compat/ alone, so every URL here +// stays put when the next release ships. +import type { GetStaticPaths } from "astro"; +import SiteLayout from "../../../components/SiteLayout.astro"; +import CompatOverview from "../../../components/CompatOverview.astro"; +import { getCompatIndex, getCompatRelease, getServiceNaming } from "../../../lib/compat-data"; + +export const getStaticPaths: GetStaticPaths = async () => { + const index = await getCompatIndex(); + return index.slice(1).map((entry) => ({ params: { tag: entry.tag } })); +}; + +const { tag } = Astro.params as { tag: string }; +const [release, index, naming] = await Promise.all([getCompatRelease(tag), getCompatIndex(), getServiceNaming()]); +if (!release) throw new Error(`compat: no report for ${tag}`); +--- + + + + diff --git a/src/pages/compat/index.astro b/src/pages/compat/index.astro new file mode 100644 index 0000000..7012f51 --- /dev/null +++ b/src/pages/compat/index.astro @@ -0,0 +1,16 @@ +--- +import SiteLayout from "../../components/SiteLayout.astro"; +import CompatOverview from "../../components/CompatOverview.astro"; +import CompatEmpty from "../../components/CompatEmpty.astro"; +import { getCompatIndex, getLatestCompatRelease, getServiceNaming } from "../../lib/compat-data"; + +const [release, index, naming] = await Promise.all([getLatestCompatRelease(), getCompatIndex(), getServiceNaming()]); +--- + + + {release ? : } + diff --git a/src/pages/compat/reason/[code].astro b/src/pages/compat/reason/[code].astro new file mode 100644 index 0000000..51e4620 --- /dev/null +++ b/src/pages/compat/reason/[code].astro @@ -0,0 +1,157 @@ +--- +// Every result with one reason, for the newest release, grouped by service. The overview's +// reason cards link here, so each "See them all" is a complete static list that works without +// script and that site search can index. The explorer does the same with any combination of +// filters; this is the fixed, linkable answer to "which ones, and why". +import type { GetStaticPaths } from "astro"; +import { ArrowLeft, ListFilter } from "@lucide/astro"; +import SiteLayout from "../../../components/SiteLayout.astro"; +import ServiceIcon from "../../../components/ServiceIcon.astro"; +import Chip from "../../../components/Chip.astro"; +import CompatIconSprite from "../../../components/CompatIconSprite.astro"; +import CompatResultDetail from "../../../components/CompatResultDetail.astro"; +import { getLatestCompatRelease, getServiceNaming } from "../../../lib/compat-data"; +import { + WHOSE_LABELS, + WHOSE_TONE, + groupResults, + reasonByCode, + reasonCount, + reasonsInUse, + serviceDisplayName, + suiteShort, + supportServiceId, + testAnchor, +} from "../../../lib/compat-report"; + +export const getStaticPaths: GetStaticPaths = async () => { + const release = await getLatestCompatRelease(); + return release ? reasonsInUse(release.report).map((reason) => ({ params: { code: reason.code } })) : []; +}; + +const { code } = Astro.params as { code: string }; +const [release, naming] = await Promise.all([getLatestCompatRelease(), getServiceNaming()]); +const report = release!.report; +const reason = report.reasons.find((r) => r.code === code)!; +const total = reasonCount(report, code); +const withEvidence = reason.whose === "overcast"; +const reasons = reasonByCode(report); + +// Tests that have this reason in at least one suite, by service. +const services = report.services + .map((service) => { + const name = serviceDisplayName(service.id, naming); + const tests = service.groups.flatMap((group) => + group.tests + .map((test) => ({ group: group.id, test, suites: report.suites.filter((suite) => test.results[suite.id]?.reason === code) })) + .filter((row) => row.suites.length > 0), + ); + const untested = code === "untested" ? (service.untested ?? []) : []; + return { service, name, tests, untested }; + }) + .filter((entry) => entry.tests.length > 0 || entry.untested.length > 0) + .sort((a, b) => a.name.localeCompare(b.name)); +--- + + + +
+ + Compatibility report {report.version} + +

{reason.label}

+

{reason.summary}

+
+ {WHOSE_LABELS[reason.whose].title} + + {total} {code === "untested" ? "operations" : "results"} in {services.length} {services.length === 1 ? "service" : "services"} + + + Narrow this down + +
+ + + + { + services.map(({ service, name, tests, untested }) => ( +
+

+ + + {name} + +

+ {tests.length > 0 && ( +
    + {tests.map(({ group, test, suites }) => ( +
  • +
    + + {test.name} + + {test.op !== test.name && {test.op}} + {group} + + {suites.map((suite) => ( + + {suiteShort(suite.id)} + + ))} + +
    + {withEvidence && ( +
    + Evidence +
    + {groupResults(test, suites, reasons).map((shared) => ( + + ))} +
    +
    + )} +
    • + ))} +
+ )} + {untested.length > 0 && ( +
    + {untested.map((gap) => ( +
  • + {gap.op} + + {gap.code} + + {gap.detail} +
    • + ))} +
+ )} +
+ )) + } +
+ diff --git a/src/pages/sitemap.xml.ts b/src/pages/sitemap.xml.ts index 05b6579..1daefa2 100644 --- a/src/pages/sitemap.xml.ts +++ b/src/pages/sitemap.xml.ts @@ -1,6 +1,8 @@ import type { APIRoute } from "astro"; import { getCollection, type CollectionEntry } from "astro:content"; import { SITE_PAGE_PATHS } from "../lib/site-pages"; +import { getCompatIndex, getLatestCompatRelease } from "../lib/compat-data"; +import { reasonsInUse } from "../lib/compat-report"; function escapeXml(value: string): string { return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); @@ -13,9 +15,20 @@ export const GET: APIRoute = async ({ site }) => { // same page in another format, and listing both would put two URLs per doc in front of a // crawler. /llms.txt is absent for the same reason — it indexes this list, it isn't a // page in it. robots.txt points at it instead. + // The compatibility report's generated pages: one per service and per reason for the newest + // release, and one overview per older release. /compat/ itself is in SITE_PAGE_PATHS. + const compat = await getLatestCompatRelease(); + const compatRoutes = compat + ? [ + ...compat.report.services.map((service) => `/compat/${service.id}/`), + ...reasonsInUse(compat.report).map((reason) => `/compat/reason/${reason.code}/`), + ...(await getCompatIndex()).slice(1).map((entry) => `/compat/history/${entry.tag}/`), + ] + : []; const routes = [ ...SITE_PAGE_PATHS, ...docs.map((doc) => `/${doc.data.slug}/`), + ...compatRoutes, ].sort(); const urls = routes diff --git a/src/styles/global.css b/src/styles/global.css index fc0abcf..37c70c0 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -766,6 +766,159 @@ kbd { } +/* Compatibility report (/compat/). A cell is one suite's result for one test: an icon and a + * word on a tint, the same colour-mix-into-card pairing as so every tone clears 4.5:1 + * on either theme. CompatCell and the explorer set `data-tone`. */ +.compat-cell { + --cell-tone: var(--oc-muted); + --cell-bg: color-mix(in srgb, var(--cell-tone) 14%, var(--oc-card)); + --cell-fg: var(--cell-tone); + display: inline-flex; + align-items: center; + gap: 0.3rem; + min-width: 4.25rem; + padding: 0.125rem 0.4rem; + border-radius: 4px; + background: var(--cell-bg); + color: var(--cell-fg); + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + line-height: 1.25rem; + white-space: nowrap; +} + +.compat-cell[data-tone="success"] { --cell-tone: var(--oc-success); } +.compat-cell[data-tone="warning"] { --cell-tone: var(--oc-warning); } +.compat-cell[data-tone="danger"] { --cell-tone: var(--oc-danger); } +.compat-cell[data-tone="accent"] { --cell-tone: var(--oc-accent); } +/* As on : the grey tint keeps its hue, and the label steps up to --oc-body to clear 4.5:1. */ +.compat-cell[data-tone="muted"] { --cell-fg: var(--oc-body); } + +.compat-cell .icon { + width: 0.8125rem; + height: 0.8125rem; + flex-shrink: 0; +} + +.compat-cell-empty { + justify-content: center; + background: transparent; + color: var(--oc-muted); +} + +/* The matrix and the per-service tables scroll sideways on a phone; the first column (the + * service or the test) stays pinned so a row never loses its name. */ +.compat-table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + font-size: 0.875rem; +} + +.compat-table th, +.compat-table td { + padding: 0.5rem 0.625rem; + border-bottom: 1px solid var(--oc-line); + text-align: left; + vertical-align: middle; +} + +.compat-table thead th { + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + font-weight: 400; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--oc-muted); + white-space: nowrap; +} + +.compat-table tbody tr:last-child > * { + border-bottom: 0; +} + +.compat-table tbody tr[data-detail] > td { + padding-top: 0; +} + +.compat-table :is(th, td):first-child { + position: sticky; + left: 0; + z-index: 1; + background: var(--oc-card); + box-shadow: 1px 0 0 var(--oc-line); +} + +.compat-table tbody tr:hover > :is(th, td) { + background: color-mix(in srgb, var(--oc-accent-soft) 60%, var(--oc-card)); +} + +.compat-table tr:target > :is(th, td) { + background: var(--oc-accent-soft); +} + +/* A matrix cell: pass/ran over a hairline meter tinted by the rate. */ +.compat-rate { + display: flex; + flex-direction: column; + gap: 0.25rem; + min-width: 3.75rem; + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + color: var(--oc-body); + text-decoration: none; +} + +.compat-rate .coverage-meter > span { + background: var(--rate-tone, var(--oc-accent)); +} + +/* Facet toggles on the explorer and the service pages: the support matrix's pressed-button + * pattern, with a count after the label. */ +.compat-toggle { + display: inline-flex; + align-items: center; + gap: 0.375rem; + padding: 0.3rem 0.6rem; + border: 1px solid var(--oc-line); + border-radius: var(--oc-radius-control); + background: var(--oc-card); + color: var(--oc-body); + font-family: var(--font-mono); + font-size: var(--oc-text-2xs); + cursor: pointer; +} + +.compat-toggle:hover { + background: var(--oc-accent-soft); +} + +.compat-toggle[aria-pressed="true"] { + border-color: var(--oc-accent); + background: var(--oc-accent-soft); + color: var(--oc-text); +} + +/* A facet that would leave nothing. Said with a dashed outline rather than by fading: faded + * text at 12px falls under 4.5:1 in every theme. */ +.compat-toggle[data-empty] { + border-style: dashed; + background: transparent; +} + +/* An explorer cell outside the current filter: kept for context, set back the same way. */ +.compat-cell[data-dim] { + --cell-bg: transparent; + --cell-fg: var(--oc-body); + outline: 1px dashed var(--oc-line); + outline-offset: -1px; +} + +.compat-toggle .compat-count { + opacity: 0.75; +} + + @font-face { font-family: "JetBrains Mono"; src: url("/fonts/JetBrainsMono-Regular.ttf") format("truetype");