From ff2a6a83474d339474ad3625c27db02e9ea0b245 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 20:29:39 +0000 Subject: [PATCH] Agent authoring benchmark (#208) benchmarks/authoring/ measures whether an agent following SKILL.md reaches FINALIZE PASS on attempt 1 with the deck its prompt asked for. Five frozen prompts carry semantic keys (format, slide range, required entities, minimum chart, logo, donut or image counts). benchmark.mjs check validates the suite; score runs finalize on each frozen candidate and writes a row per case with the failed stage, the misses, the top three diagnostic codes, and the runner's repairs and minutes. It never launches a model. Five hand-built references prove the wiring and are marked evidence: false. README gives the fair-run protocol. ponytail: structural semantic keys only (format, counts, entities); a fact-level check of the prompt's numbers waits for the first real runs. Closes #208 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CiXonXtVxd5FHk4XqMzY8f --- .gitignore | 5 + CHANGELOG.md | 1 + CONTRIBUTING.md | 1 + benchmarks/authoring/README.md | 37 ++ benchmarks/authoring/benchmark.mjs | 95 ++++ .../authoring/inputs/landscape/bluefin.svg | 1 + .../authoring/inputs/landscape/corvid.svg | 1 + .../authoring/inputs/landscape/delta-loom.svg | 1 + .../authoring/inputs/landscape/emberly.svg | 1 + .../authoring/inputs/landscape/fathom.svg | 1 + .../authoring/inputs/landscape/gridline.svg | 1 + .../authoring/inputs/landscape/halcyon.svg | 1 + .../authoring/inputs/landscape/ironleaf.svg | 1 + .../authoring/inputs/landscape/juniper.svg | 1 + .../authoring/inputs/landscape/kestrel.svg | 1 + .../authoring/inputs/landscape/lumen.svg | 1 + .../authoring/inputs/landscape/northwind.svg | 1 + benchmarks/authoring/manifest.json | 13 + benchmarks/authoring/prompts/carousel.md | 9 + benchmarks/authoring/prompts/landscape.md | 5 + benchmarks/authoring/prompts/one-pager.md | 9 + benchmarks/authoring/prompts/pitch.md | 9 + benchmarks/authoring/prompts/qbr.md | 9 + .../authoring/reference/carousel/model.json | 132 ++++++ .../authoring/reference/landscape/model.json | 178 +++++++ .../authoring/reference/one-pager/model.json | 66 +++ .../authoring/reference/pitch/model.json | 445 ++++++++++++++++++ benchmarks/authoring/reference/qbr/model.json | 419 +++++++++++++++++ benchmarks/authoring/reference/run.json | 13 + benchmarks/authoring/results/.gitkeep | 0 llms.txt | 1 + package.json | 3 +- test/benchmark.test.mjs | 78 +++ 33 files changed, 1539 insertions(+), 1 deletion(-) create mode 100644 benchmarks/authoring/README.md create mode 100644 benchmarks/authoring/benchmark.mjs create mode 100644 benchmarks/authoring/inputs/landscape/bluefin.svg create mode 100644 benchmarks/authoring/inputs/landscape/corvid.svg create mode 100644 benchmarks/authoring/inputs/landscape/delta-loom.svg create mode 100644 benchmarks/authoring/inputs/landscape/emberly.svg create mode 100644 benchmarks/authoring/inputs/landscape/fathom.svg create mode 100644 benchmarks/authoring/inputs/landscape/gridline.svg create mode 100644 benchmarks/authoring/inputs/landscape/halcyon.svg create mode 100644 benchmarks/authoring/inputs/landscape/ironleaf.svg create mode 100644 benchmarks/authoring/inputs/landscape/juniper.svg create mode 100644 benchmarks/authoring/inputs/landscape/kestrel.svg create mode 100644 benchmarks/authoring/inputs/landscape/lumen.svg create mode 100644 benchmarks/authoring/inputs/landscape/northwind.svg create mode 100644 benchmarks/authoring/manifest.json create mode 100644 benchmarks/authoring/prompts/carousel.md create mode 100644 benchmarks/authoring/prompts/landscape.md create mode 100644 benchmarks/authoring/prompts/one-pager.md create mode 100644 benchmarks/authoring/prompts/pitch.md create mode 100644 benchmarks/authoring/prompts/qbr.md create mode 100644 benchmarks/authoring/reference/carousel/model.json create mode 100644 benchmarks/authoring/reference/landscape/model.json create mode 100644 benchmarks/authoring/reference/one-pager/model.json create mode 100644 benchmarks/authoring/reference/pitch/model.json create mode 100644 benchmarks/authoring/reference/qbr/model.json create mode 100644 benchmarks/authoring/reference/run.json create mode 100644 benchmarks/authoring/results/.gitkeep create mode 100644 test/benchmark.test.mjs diff --git a/.gitignore b/.gitignore index 64845a2..3df7863 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ templates/library-*.style.json templates/landed.index.json .decklet-host.json .decklet-history/ + +# authoring benchmark: what score writes beside each candidate +benchmarks/authoring/**/deck.html +benchmarks/authoring/**/deck.finalize.json +benchmarks/authoring/**/verify-out/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c2ccc4..07224bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## Unreleased +- **Agent authoring benchmark (#208).** `benchmarks/authoring/` scores whether an agent following SKILL.md reaches `FINALIZE PASS` on attempt 1 with the deck the prompt asked for. Five frozen prompts (a pitch, a QBR, a one-page document, a 4:5 carousel and a twelve-vendor landscape with its logo files in `inputs/`) each carry semantic keys: format, slide range, required entities and minimum chart, logo, donut or image counts. `benchmark.mjs check` validates the suite; `score ` runs `finalize` on each frozen candidate and writes one row per case: pass, the failed stage, the semantic misses, the top three diagnostic codes, repairs and minutes from the runner's `run.json`. It never launches a model. Five hand-built references prove the wiring and are marked `evidence: false`. README.md gives the fair-run protocol: same commit and prompts, an installed decklet outside this repo, attempt 1 frozen. Gate: test/benchmark.test.mjs (the manifest, a broken manifest, semantic misses, stubbed and live scoring of every reference, a missing and a broken candidate, the CLI). - **Update notice in the finalize receipt (#206).** `node bin/finalize.mjs` adds `update: {installed, available, noticeRequired, source}` to its receipt and prints one line when a newer decklet is on main. `lib/release.mjs` reads `version` from `package.json` on main with curl under a 4-second limit (git tags lag the releases), caches the answer for 24 hours under `$XDG_CACHE_HOME/decklet`, reports `unavailable` instead of failing when offline, and never installs anything; `DECKLET_NO_UPDATE_CHECK=1` turns it off. The API asks only when called with `update`, so tests and embedders make no request. The check never changes `ok`. SKILL.md Step 6 tells the agent to mention it once. Gate: test/release.test.mjs (newer, same and older versions, a failed or garbled fetch, the 24-hour cache, the off switch, and the receipt). - **Coded verify diagnostics (#201, verify half).** `verify` now returns `diagnostics` in the shape `validate` has: the contract's codes first, then one per rendered problem with its `engine`, a `subject` of `{slide, el, id}` (or `{slide, master, id}`), evidence and fixes. Eleven `parity/` codes cover overflow (`{need, have}`, `w: `), nowrap lines, line count, ink through text, a straddled container, text over text, an arrow in a fill, the canvas edge, occlusion and the counter's margin and drift, plus `parity/page-error`, `parity/crowding` (a warning), `ae/over-threshold` and `deck/network`. The codes are read from the problem strings the browser pass already writes, so the in-page measuring code is unchanged. `verify --json` prints them, `results.json` carries them (and is now written after the aggregate errors, so it holds those too), and the `finalize` receipt carries them on the verify stage. Gate: test/verify-codes.test.mjs (a static check that every problem string verify can push has a code, each code's subject and evidence, crowding, page errors, AE and the network, the receipt, and a live overflow read back through the API, `results.json` and `--json`). - **Two windows no longer lose an edit in the store.** When window B was typing as window A saved, B saved its words on top of A's model, but A's older save could reach localStorage after B's. Both windows looked right and the store held A's model without B's words, so a reload lost them. A CI probe caught this on Firefox in 12 of 180 runs under load and 0 of 60 alone. A window now watches the `storage` event: when the store holds a model it already built on, it writes its newer model back, and it writes the log back when entries are missing. This was the Firefox two-window failure on `main`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39f3840..67de1e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,7 @@ Node 22 or newer. Nothing else to install for the engine or the CLI. | HTML pages to model | `bin/import-html.mjs` | | Worked examples | `examples//brief.md` + `model.json` | | Layout library | `templates/` (then `npm run build:library`) | +| A SKILL.md change, and its measured effect | `SKILL.md`, then a run of `benchmarks/authoring/` (its README has the protocol) | ## License diff --git a/benchmarks/authoring/README.md b/benchmarks/authoring/README.md new file mode 100644 index 0000000..1684175 --- /dev/null +++ b/benchmarks/authoring/README.md @@ -0,0 +1,37 @@ +# Authoring benchmark + +Does an agent that follows `SKILL.md` reach `FINALIZE PASS` on attempt 1, with the deck the prompt asked for? Rerun it on every SKILL.md change and compare the numbers. + +## What a case checks + +`manifest.json` holds five cases: a pitch, a quarterly business review, a one-page document, a 4:5 carousel and a twelve-vendor competitive landscape. Each case has a frozen prompt in `prompts/` and semantic keys the scorer checks on the candidate's `model.json`: + +| key | the candidate must | +|---|---| +| `format` | name this format | +| `density` | (in the prompt) build at this density | +| `slides` | have between `[min, max]` slides | +| `entities` | list every one of these companies in `deck.entities` | +| `counts` | carry at least this many `chart`, `logo`, `donut` or `img` rows | + +A row passes only when `finalize` passes (validate `--strict`, create, verify `--strict`) and no semantic key misses. A deck that renders but drops a required vendor fails, and so does a faithful deck that `finalize` rejects. + +## Fair-run protocol + +1. Run every configuration on the same commit, prompt set, time limit and tool access. Record the agent and model names in `run.json`. +2. Run the agent from an installed copy of decklet (plugin or `npx skills add`), never from this repository, so it sees what users see and never sees these cases or the references. +3. Give it the prompt and, for a case with `inputs`, a copy of that folder in its working directory. +4. One complete agent invocation is attempt 1. Freeze its `model.json` (and `style.json`, if any) into `//` when it ends. Later attempts never replace it. +5. Write `run.json`: `{agent, model, cases: {: {minutes, repairs}}}`, with repairs counted from the transcript (runs of finalize or verify after the first). +6. Score it and keep the result: + +```bash +node benchmarks/authoring/benchmark.mjs check +node benchmarks/authoring/benchmark.mjs score --out benchmarks/authoring/results/--.json +``` + +The script never launches a model; the runner owns models, credentials, timing and transcripts. + +## References + +`reference/` holds one hand-built candidate per case. They prove the scorer and the cases are wired (test/benchmark.test.mjs scores them), and `run.json` marks them `"evidence": false`. They are never benchmark results. diff --git a/benchmarks/authoring/benchmark.mjs b/benchmarks/authoring/benchmark.mjs new file mode 100644 index 0000000..dbb47cc --- /dev/null +++ b/benchmarks/authoring/benchmark.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node +// decklet authoring benchmark — does an agent following SKILL.md reach FINALIZE PASS on attempt 1, with the deck the prompt asked for? +// node benchmarks/authoring/benchmark.mjs check [--manifest manifest.json] +// node benchmarks/authoring/benchmark.mjs score [--out result.json] [--no-webkit] [--no-verify] +// A run dir holds /model.json (and an optional style.json) per case plus run.json {agent, model, cases: {id: {minutes, repairs}}}. +// This script never launches a model: the runner owns models, prompts, timing and transcripts. README.md has the protocol. +import fs from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {finalize} from '../../bin/finalize.mjs'; +import {FORMATS} from '../../bin/validate.mjs'; +import {DENSITY} from '../../lib/layouts.mjs'; +import {isMain} from '../../lib/is-main.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const COUNTS = ['chart', 'logo', 'donut', 'img']; +const read = f => JSON.parse(fs.readFileSync(f, 'utf8')); + +export function check(manifest, dir = here) { + const problems = [], ids = new Set(); + if (!Array.isArray(manifest.cases) || !manifest.cases.length) return {problems: ['cases must be a non-empty array']}; + for (const c of manifest.cases) { + const id = c.id || '?', P = m => problems.push(`${id}: ${m}`); + if (!/^[a-z0-9-]+$/.test(c.id || '')) P('id must be a slug'); + if (ids.has(c.id)) P('duplicate id'); ids.add(c.id); + if (!c.prompt || !fs.existsSync(path.join(dir, c.prompt)) || !fs.readFileSync(path.join(dir, c.prompt), 'utf8').trim()) P(`prompt ${c.prompt} not found`); + if (!FORMATS.includes(c.format)) P(`format "${c.format}" is not one of ${FORMATS.join('|')}`); + if (!DENSITY[c.density]) P(`density "${c.density}" is not one of ${Object.keys(DENSITY).join('|')}`); + if (!(Array.isArray(c.slides) && c.slides.length === 2 && c.slides.every(Number.isInteger) && c.slides[0] >= 1 && c.slides[0] <= c.slides[1])) P('slides must be [min, max]'); + if (!Array.isArray(c.entities)) P('entities must be an array'); + if (!c.counts || typeof c.counts !== 'object' || Array.isArray(c.counts)) P('counts must be an object'); + else for (const k of Object.keys(c.counts)) if (!COUNTS.includes(k)) P(`counts.${k} is not one of ${COUNTS.join('|')}`); + if (c.inputs && !fs.existsSync(path.join(dir, c.inputs))) P(`inputs ${c.inputs} not found`); + } + return {problems}; +} + +const rowsOf = m => [...(m.slides || []).flatMap(s => s?.els || []), ...(m.master || [])]; +const fillMedia = m => (m.slides || []).flatMap(s => Object.values(s?.fill || {})).filter(v => v && typeof v === 'object' && !Array.isArray(v)); +const count = (m, kind) => { + const rows = rowsOf(m); + if (kind === 'chart') return rows.filter(r => r?.chart).length + (m.slides || []).filter(s => /^chart/.test(s?.template || '')).length; + if (kind === 'logo') return rows.filter(r => r?.logo != null).length + fillMedia(m).filter(v => v.logo != null).length; + if (kind === 'img') return rows.filter(r => r?.img).length + fillMedia(m).filter(v => v.img).length; + return rows.filter(r => r?.[kind] != null).length; +}; + +export function semanticMisses(c, m) { + const misses = [], n = (m.slides || []).length, fmt = m.format || 'slides'; + if (fmt !== c.format) misses.push(`format ${fmt}, wanted ${c.format}`); + if (n < c.slides[0] || n > c.slides[1]) misses.push(`slides ${n}, wanted ${c.slides[0]}-${c.slides[1]}`); + const listed = new Set((m.entities || []).flat().map(e => String(e).toLowerCase())); + for (const e of c.entities) if (!listed.has(e.toLowerCase())) misses.push(`entities missing ${e}`); + for (const [k, want] of Object.entries(c.counts)) { const got = count(m, k); if (got < want) misses.push(`${k} ${got}, wanted at least ${want}`); } + return misses; +} + +const topCodes = receipt => { + const n = new Map(); + for (const s of receipt.stages) for (const d of s.diagnostics || []) n.set(d.code, (n.get(d.code) || 0) + 1); + return [...n].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, 3).map(([c]) => c); +}; + +export async function score(runDir, {manifestPath = path.join(here, 'manifest.json'), verifyFn = null, webkit = 'auto'} = {}) { + const manifest = read(manifestPath), runFile = path.join(runDir, 'run.json'), run = fs.existsSync(runFile) ? read(runFile) : {}; + const rows = []; + for (const c of manifest.cases) { + const dir = path.join(runDir, c.id), modelFile = path.join(dir, 'model.json'), styleFile = path.join(dir, 'style.json'); + const meta = {repairs: run.cases?.[c.id]?.repairs ?? null, minutes: run.cases?.[c.id]?.minutes ?? null}; + if (!fs.existsSync(modelFile)) { rows.push({case: c.id, status: 'missing', pass: false, finalize: false, misses: ['no model.json'], topCodes: [], ...meta}); continue; } + const receipt = await finalize(modelFile, {style: fs.existsSync(styleFile) ? styleFile : null, out: path.join(dir, 'deck.html'), webkit, ...(verifyFn ? {verifyFn} : {}), log: () => {}}); + const misses = semanticMisses(c, read(modelFile)), failed = receipt.stages.find(s => s.status !== 'pass'); + rows.push({case: c.id, status: receipt.ok ? 'finalized' : 'failed', pass: receipt.ok && !misses.length, finalize: receipt.ok, ...(failed ? {failedStage: failed.stage} : {}), misses, topCodes: topCodes(receipt), ...meta}); + } + return {benchmark: manifest.benchmark, version: manifest.version, agent: run.agent ?? null, model: run.model ?? null, evidence: run.evidence ?? run.agent !== 'reference', + engine: JSON.parse(fs.readFileSync(path.join(here, '../../package.json'), 'utf8')).version, scoredAt: new Date().toISOString(), + rows, summary: {passed: rows.filter(r => r.pass).length, total: rows.length}}; +} + +if (isMain(import.meta.url)) { + const [cmd, ...a] = process.argv.slice(2), flag = k => a.includes(k), opt = k => a.includes(k) ? a[a.indexOf(k) + 1] : null; + if (cmd === 'check') { + const mp = opt('--manifest') || path.join(here, 'manifest.json'), {problems} = check(read(mp), path.dirname(path.resolve(mp))); + for (const p of problems) console.error('ERROR ' + p); + console.log(problems.length ? `manifest has ${problems.length} problem(s)` : `manifest ok: ${read(mp).cases.length} cases`); + process.exit(problems.length ? 1 : 0); + } + const runDir = a.find(x => !x.startsWith('--') && x !== opt('--out')); + if (cmd !== 'score' || !runDir) { console.error('usage: node benchmarks/authoring/benchmark.mjs check [--manifest file] | score [--out file] [--no-webkit] [--no-verify]'); process.exit(2); } + const noVerify = async () => ({ok: true, errors: [], warnings: [], skipped: ['layout parity + AE: --no-verify'], parity: [], diagnostics: []}); + const r = await score(runDir, {webkit: flag('--no-webkit') ? false : 'auto', ...(flag('--no-verify') ? {verifyFn: noVerify} : {})}); + for (const x of r.rows) console.log(`${x.pass ? 'pass' : 'FAIL'} ${x.case.padEnd(10)} ${x.status.padEnd(9)} ${x.misses.join('; ') || '-'} ${x.topCodes.join(', ')}`); + console.log(`${r.summary.passed}/${r.summary.total} passed${r.evidence ? '' : ' (not evidence: reference candidates)'}`); + if (opt('--out')) fs.writeFileSync(opt('--out'), JSON.stringify(r, null, 1) + '\n'); +} diff --git a/benchmarks/authoring/inputs/landscape/bluefin.svg b/benchmarks/authoring/inputs/landscape/bluefin.svg new file mode 100644 index 0000000..4a3da15 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/bluefin.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/corvid.svg b/benchmarks/authoring/inputs/landscape/corvid.svg new file mode 100644 index 0000000..f27099c --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/corvid.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/delta-loom.svg b/benchmarks/authoring/inputs/landscape/delta-loom.svg new file mode 100644 index 0000000..9eef91f --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/delta-loom.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/emberly.svg b/benchmarks/authoring/inputs/landscape/emberly.svg new file mode 100644 index 0000000..ae7bc46 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/emberly.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/fathom.svg b/benchmarks/authoring/inputs/landscape/fathom.svg new file mode 100644 index 0000000..fbed0e6 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/fathom.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/gridline.svg b/benchmarks/authoring/inputs/landscape/gridline.svg new file mode 100644 index 0000000..5b92452 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/gridline.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/halcyon.svg b/benchmarks/authoring/inputs/landscape/halcyon.svg new file mode 100644 index 0000000..7272b94 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/halcyon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/ironleaf.svg b/benchmarks/authoring/inputs/landscape/ironleaf.svg new file mode 100644 index 0000000..c22623e --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/ironleaf.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/juniper.svg b/benchmarks/authoring/inputs/landscape/juniper.svg new file mode 100644 index 0000000..4f7abde --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/juniper.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/kestrel.svg b/benchmarks/authoring/inputs/landscape/kestrel.svg new file mode 100644 index 0000000..6c22599 --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/kestrel.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/lumen.svg b/benchmarks/authoring/inputs/landscape/lumen.svg new file mode 100644 index 0000000..972ceca --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/lumen.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/inputs/landscape/northwind.svg b/benchmarks/authoring/inputs/landscape/northwind.svg new file mode 100644 index 0000000..79e52bd --- /dev/null +++ b/benchmarks/authoring/inputs/landscape/northwind.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/benchmarks/authoring/manifest.json b/benchmarks/authoring/manifest.json new file mode 100644 index 0000000..9d20497 --- /dev/null +++ b/benchmarks/authoring/manifest.json @@ -0,0 +1,13 @@ +{ + "benchmark": "decklet-authoring", + "version": 1, + "cases": [ + {"id": "pitch", "prompt": "prompts/pitch.md", "format": "slides", "density": "speaker", "slides": [8, 12], "entities": [], "counts": {"chart": 1}}, + {"id": "qbr", "prompt": "prompts/qbr.md", "format": "slides", "density": "reading", "slides": [6, 10], "entities": [], "counts": {"chart": 1}}, + {"id": "one-pager", "prompt": "prompts/one-pager.md", "format": "document-letter", "density": "reading", "slides": [1, 1], "entities": [], "counts": {"chart": 1}}, + {"id": "carousel", "prompt": "prompts/carousel.md", "format": "carousel-4x5", "density": "speaker", "slides": [5, 8], "entities": [], "counts": {}}, + {"id": "landscape", "prompt": "prompts/landscape.md", "format": "slides", "density": "speaker", "slides": [2, 4], + "entities": ["Northwind", "Bluefin", "Corvid", "Delta Loom", "Emberly", "Fathom", "Gridline", "Halcyon", "Ironleaf", "Juniper", "Kestrel", "Lumen"], + "counts": {"logo": 12}, "inputs": "inputs/landscape"} + ] +} diff --git a/benchmarks/authoring/prompts/carousel.md b/benchmarks/authoring/prompts/carousel.md new file mode 100644 index 0000000..37bb008 --- /dev/null +++ b/benchmarks/authoring/prompts/carousel.md @@ -0,0 +1,9 @@ +Use the decklet skill to make a LinkedIn carousel of 5 to 8 cards on the `carousel-4x5` format at speaker density, saved as `deck.html` next to `model.json`. + +Topic: three habits of fast engineering teams. +- Small changes: teams that merge under 200 lines ship twice as often. +- Fast review: a first review inside four hours halves cycle time. +- Green main: a main branch that stays green saves about a day a week. +- Close with a card asking the reader to pick one habit and measure it for a month. + +Give every card one graphic. Finish with `finalize` passing. diff --git a/benchmarks/authoring/prompts/landscape.md b/benchmarks/authoring/prompts/landscape.md new file mode 100644 index 0000000..3ce191c --- /dev/null +++ b/benchmarks/authoring/prompts/landscape.md @@ -0,0 +1,5 @@ +Use the decklet skill to make a competitive landscape deck of 2 to 4 slides on the 16:9 `slides` format at speaker density, saved as `deck.html` next to `model.json`. + +The market is workflow tools for operations teams. The twelve vendors are Northwind, Bluefin, Corvid, Delta Loom, Emberly, Fathom, Gridline, Halcyon, Ironleaf, Juniper, Kestrel and Lumen. Their logos are the SVG files in `inputs/landscape/`, one per vendor, named after it. Every vendor appears with its logo. List all twelve in `deck.entities`. + +Group them however the deck reads best. Finish with `finalize` passing. diff --git a/benchmarks/authoring/prompts/one-pager.md b/benchmarks/authoring/prompts/one-pager.md new file mode 100644 index 0000000..9d2ba06 --- /dev/null +++ b/benchmarks/authoring/prompts/one-pager.md @@ -0,0 +1,9 @@ +Use the decklet skill to make a one-page document on the `document-letter` format at reading density, saved as `deck.html` next to `model.json`. + +Subject: results of the onboarding checklist pilot. +- 40 engineers hired in August and September used a guided checklist. +- Median time to first shipped change fell from 9 working days to 4. +- Next step: extend the checklist to the support team in November and measure the same interval. +- Source: onboarding tracker. + +Include the before/after numbers as a chart and cite the source. Finish with `finalize` passing. diff --git a/benchmarks/authoring/prompts/pitch.md b/benchmarks/authoring/prompts/pitch.md new file mode 100644 index 0000000..106a0cd --- /dev/null +++ b/benchmarks/authoring/prompts/pitch.md @@ -0,0 +1,9 @@ +Use the decklet skill to make a presented investor pitch deck, 8 to 12 slides on the 16:9 `slides` format at speaker density, saved as `deck.html` next to `model.json`. + +Company: Tidewater, a scheduling app for loading docks at regional warehouses. +- Problem: trucks wait an average of 2.4 hours for a dock door; detention fees cost a mid-size warehouse about $310,000 a year. +- Product: drivers book a door slot from their phone; the warehouse sees every arrival on one board. +- Traction: 38 warehouses live, monthly recurring revenue up from $41k in January to $186k in September. +- Ask: $6M seed to reach 250 warehouses in 18 months. + +Show the revenue growth as a chart. Finish with `finalize` passing. diff --git a/benchmarks/authoring/prompts/qbr.md b/benchmarks/authoring/prompts/qbr.md new file mode 100644 index 0000000..a8a7ecd --- /dev/null +++ b/benchmarks/authoring/prompts/qbr.md @@ -0,0 +1,9 @@ +Use the decklet skill to make a quarterly business review that will be read without a speaker: 6 to 10 slides on the 16:9 `slides` format at reading density, saved as `deck.html` next to `model.json`. + +Q3 figures for the support team: +- Tickets: 18,400 (Q2: 16,900). First response median: 2h10m (Q2: 3h05m). +- CSAT: 91% (Q2: 88%). Backlog at quarter end: 640 (Q2: 1,120). +- Headcount: 22 agents, unchanged. Self-serve deflection rose from 18% to 27% after the help-centre rewrite. +- Q4 goals: first response under 2 hours, CSAT 92%, launch chat for enterprise accounts. + +Chart at least one quarter-over-quarter comparison. Finish with `finalize` passing. diff --git a/benchmarks/authoring/reference/carousel/model.json b/benchmarks/authoring/reference/carousel/model.json new file mode 100644 index 0000000..83ddc07 --- /dev/null +++ b/benchmarks/authoring/reference/carousel/model.json @@ -0,0 +1,132 @@ +{ + "format": "carousel-4x5", + "density": "speaker", + "title": "Three habits", + "slides": [ + { + "els": [ + { + "x": 90, + "y": 120, + "w": 900, + "role": "Title", + "text": "Three habits of fast teams" + }, + { + "x": 90, + "y": 420, + "w": 900, + "role": "Body", + "text": "Swipe for the numbers behind each one." + }, + { + "x": 340, + "y": 700, + "w": 400, + "donut": 100, + "color": "var(--accent)" + } + ] + }, + { + "els": [ + { + "x": 90, + "y": 120, + "w": 900, + "role": "Title", + "text": "Small changes" + }, + { + "x": 90, + "y": 420, + "w": 900, + "role": "Body", + "text": "Teams that merge under 200 lines ship twice as often." + }, + { + "x": 340, + "y": 700, + "w": 400, + "donut": 72, + "color": "var(--accent)" + } + ] + }, + { + "els": [ + { + "x": 90, + "y": 120, + "w": 900, + "role": "Title", + "text": "Fast review" + }, + { + "x": 90, + "y": 420, + "w": 900, + "role": "Body", + "text": "A first review inside four hours halves cycle time." + }, + { + "x": 340, + "y": 700, + "w": 400, + "donut": 58, + "color": "var(--accent)" + } + ] + }, + { + "els": [ + { + "x": 90, + "y": 120, + "w": 900, + "role": "Title", + "text": "Green main" + }, + { + "x": 90, + "y": 420, + "w": 900, + "role": "Body", + "text": "A main branch that stays green saves a day a week." + }, + { + "x": 340, + "y": 700, + "w": 400, + "donut": 81, + "color": "var(--accent)" + } + ] + }, + { + "els": [ + { + "x": 90, + "y": 120, + "w": 900, + "role": "Title", + "text": "Start with one" + }, + { + "x": 90, + "y": 420, + "w": 900, + "role": "Body", + "text": "Pick the habit your team misses most and measure it for a month." + }, + { + "x": 340, + "y": 700, + "w": 400, + "donut": 100, + "color": "var(--accent)" + } + ] + } + ] +} diff --git a/benchmarks/authoring/reference/landscape/model.json b/benchmarks/authoring/reference/landscape/model.json new file mode 100644 index 0000000..ee7cb8d --- /dev/null +++ b/benchmarks/authoring/reference/landscape/model.json @@ -0,0 +1,178 @@ +{ + "format": "slides", + "density": "speaker", + "title": "Workflow tools", + "entities": [ + "Northwind", + "Bluefin", + "Corvid", + "Delta Loom", + "Emberly", + "Fathom", + "Gridline", + "Halcyon", + "Ironleaf", + "Juniper", + "Kestrel", + "Lumen" + ], + "assets": { + "l0": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMyNTYzZWIiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxMiIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l1": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMxNmEzNGEiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxNSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l2": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiNkYjI3NzciLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxOCIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l3": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiNlYTU4MGMiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIyMSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l4": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiM3YzNhZWQiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxMiIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l5": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMwODkxYjIiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxNSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l6": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMyNTYzZWIiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxOCIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l7": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMxNmEzNGEiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIyMSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l8": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiNkYjI3NzciLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxMiIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l9": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiNlYTU4MGMiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxNSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l10": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiM3YzNhZWQiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIxOCIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==", + "l11": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI2NCIgaGVpZ2h0PSI2NCI+PHJlY3Qgd2lkdGg9IjY0IiBoZWlnaHQ9IjY0IiByeD0iMTIiIGZpbGw9IiMwODkxYjIiLz48Y2lyY2xlIGN4PSIzMiIgY3k9IjMyIiByPSIyMSIgZmlsbD0iI2ZmZiIvPjwvc3ZnPg==" + }, + "slides": [ + { + "textOnly": true, + "els": [ + { + "x": 60, + "y": 200, + "w": 840, + "role": "Title", + "text": "Who builds workflow tools" + } + ] + }, + { + "els": [ + { + "x": 60, + "y": 60, + "w": 840, + "role": "H1", + "text": "Twelve vendors, three camps" + }, + { + "logo": "#l0", + "name": "Northwind", + "x": 60, + "y": 150, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l1", + "name": "Bluefin", + "x": 350, + "y": 150, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l2", + "name": "Corvid", + "x": 640, + "y": 150, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l3", + "name": "Delta Loom", + "x": 60, + "y": 230, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l4", + "name": "Emberly", + "x": 350, + "y": 230, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l5", + "name": "Fathom", + "x": 640, + "y": 230, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l6", + "name": "Gridline", + "x": 60, + "y": 310, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l7", + "name": "Halcyon", + "x": 350, + "y": 310, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l8", + "name": "Ironleaf", + "x": 640, + "y": 310, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l9", + "name": "Juniper", + "x": 60, + "y": 390, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l10", + "name": "Kestrel", + "x": 350, + "y": 390, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + }, + { + "logo": "#l11", + "name": "Lumen", + "x": 640, + "y": 390, + "h": 40, + "col": 40, + "plate": "none", + "aspect": 1 + } + ] + } + ] +} diff --git a/benchmarks/authoring/reference/one-pager/model.json b/benchmarks/authoring/reference/one-pager/model.json new file mode 100644 index 0000000..b5611ea --- /dev/null +++ b/benchmarks/authoring/reference/one-pager/model.json @@ -0,0 +1,66 @@ +{ + "format": "document-letter", + "density": "reading", + "title": "Pilot results", + "counter": 0, + "slides": [ + { + "els": [ + { + "x": 48, + "y": 48, + "w": 720, + "role": "Supertitle", + "text": "Pilot results · Q3" + }, + { + "x": 48, + "y": 72, + "w": 720, + "role": "Title", + "text": "Onboarding fell from nine days to four" + }, + { + "x": 48, + "y": 260, + "w": 720, + "role": "Body", + "text": "Forty new hires used the guided checklist in August and September. Median time to first shipped change fell from nine working days to four." + }, + { + "x": 48, + "y": 370, + "w": 720, + "h": 280, + "chart": { + "mark": "bar", + "data": [ + { + "label": "Before", + "value": 9 + }, + { + "label": "Pilot", + "value": 4 + } + ] + } + }, + { + "x": 48, + "y": 700, + "w": 720, + "role": "Body", + "text": "Next: extend the checklist to the support team in November and measure the same interval." + }, + { + "x": 48, + "y": 960, + "w": 720, + "role": "Caption", + "text": "Source: onboarding tracker, 40 hires, Aug–Sep." + } + ] + } + ] +} diff --git a/benchmarks/authoring/reference/pitch/model.json b/benchmarks/authoring/reference/pitch/model.json new file mode 100644 index 0000000..0c95380 --- /dev/null +++ b/benchmarks/authoring/reference/pitch/model.json @@ -0,0 +1,445 @@ +{ + "$schema": "../../../../home/user/wt-bench/schemas/model.schema.json", + "title": "Meridian Freight — PLACEHOLDER deck", + "format": "slides", + "w": 960, + "h": 540, + "density": "speaker", + "styles": { + "margin": 60 + }, + "master": [ + { + "id": "foot", + "footer": 1, + "right": 60, + "y": 506, + "w": "auto", + "role": "Label", + "nowrap": 1, + "text": "Meridian Freight · PLACEHOLDER" + } + ], + "slides": [ + { + "name": "cover", + "layout": "cover", + "els": [ + { + "slot": "supertitle", + "text": "MERIDIAN FREIGHT · PLACEHOLDER DECK" + }, + { + "slot": "title", + "text": "Replace with the deck's one-line promise." + }, + { + "slot": "body", + "text": "Replace with who this is for and what they get." + }, + { + "slot": "caption", + "text": "Replace · presenter, audience, date" + } + ] + }, + { + "name": "agenda", + "layout": "agenda", + "els": [ + { + "slot": "supertitle", + "text": "AGENDA" + }, + { + "slot": "title", + "text": "Replace with what the deck covers." + }, + { + "slot": "n1", + "text": "01" + }, + { + "slot": "item1", + "text": "Replace · section 1" + }, + { + "slot": "n2", + "text": "02" + }, + { + "slot": "item2", + "text": "Replace · section 2" + }, + { + "slot": "n3", + "text": "03" + }, + { + "slot": "item3", + "text": "Replace · section 3" + } + ] + }, + { + "name": "points", + "layout": "bullets", + "textOnly": true, + "els": [ + { + "slot": "supertitle", + "text": "POINTS" + }, + { + "slot": "title", + "text": "Replace with the claim these points carry." + }, + { + "slot": "b1", + "text": "Replace · point 1" + }, + { + "slot": "b2", + "text": "Replace · point 2" + }, + { + "slot": "b3", + "text": "Replace · point 3" + } + ] + }, + { + "name": "numbers", + "template": "area-bubbles-speaker", + "fill": { + "t1": "NUMBERS", + "t2": "Replace with what the numbers say.", + "t3": "30", + "t4": "Replace · metric 1", + "t5": "20", + "t6": "Replace · metric 2", + "t7": "10", + "t8": "Replace · metric 3", + "data": [ + { + "label": "Replace · metric 1", + "value": 30 + }, + { + "label": "Replace · metric 2", + "value": 20 + }, + { + "label": "Replace · metric 3", + "value": 10 + } + ] + } + }, + { + "name": "trend", + "layout": "chart", + "els": [ + { + "slot": "supertitle", + "text": "TREND" + }, + { + "slot": "title", + "text": "Replace with what the series shows." + }, + { + "slot": "chart", + "chart": { + "mark": "bar", + "data": [ + { + "label": "One", + "value": 10 + }, + { + "label": "Two", + "value": 20 + }, + { + "label": "Three", + "value": 30 + }, + { + "label": "Four", + "value": 40 + } + ] + } + }, + { + "slot": "takeaway", + "text": "Replace with the takeaway. The bars above are a placeholder ramp, not data." + } + ] + }, + { + "name": "process", + "layout": "process-steps", + "textOnly": true, + "els": [ + { + "slot": "supertitle", + "text": "PROCESS" + }, + { + "slot": "title", + "text": "Replace with the name of the process." + }, + { + "slot": "n1", + "text": "01" + }, + { + "slot": "step1", + "text": "Replace · step 1" + }, + { + "slot": "n2", + "text": "02" + }, + { + "slot": "step2", + "text": "Replace · step 2" + }, + { + "slot": "n3", + "text": "03" + }, + { + "slot": "step3", + "text": "Replace · step 3" + } + ] + }, + { + "name": "figure", + "layout": "diagram", + "els": [ + { + "slot": "supertitle", + "text": "FIGURE" + }, + { + "slot": "title", + "text": "Replace with the name of the figure." + }, + { + "id": "a", + "x": 68, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--card)", + "bd": "1px solid var(--fg)", + "radius": 8 + }, + { + "x": 84, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace A" + }, + { + "x": 84, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "id": "b", + "x": 380, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--card)", + "bd": "1px solid var(--fg)", + "radius": 8 + }, + { + "x": 396, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace B" + }, + { + "x": 396, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "id": "c", + "x": 692, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--box)", + "bd": "2px solid var(--accent)", + "radius": 8 + }, + { + "x": 708, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace C" + }, + { + "x": 708, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "x": 268, + "y": 280, + "line": [ + 380, + 280 + ], + "h": 2.5, + "bg": "var(--fg)", + "from": "a", + "to": "b", + "arrow": "end", + "head": "triangle" + }, + { + "x": 580, + "y": 280, + "line": [ + 692, + 280 + ], + "h": 3, + "bg": "var(--accent)", + "from": "b", + "to": "c", + "arrow": "end", + "head": "triangle" + }, + { + "slot": "caption", + "text": "Replace with the sentence this figure proves." + } + ] + }, + { + "name": "options", + "layout": "comparison", + "textOnly": true, + "els": [ + { + "slot": "supertitle", + "text": "OPTIONS" + }, + { + "slot": "title", + "text": "Replace with the choice being made." + }, + { + "slot": "left-head", + "text": "Replace · option A" + }, + { + "slot": "right-head", + "text": "Replace · option B" + } + ] + }, + { + "name": "timeline", + "layout": "timeline", + "els": [ + { + "slot": "supertitle", + "text": "TIMELINE" + }, + { + "slot": "title", + "text": "Replace with what the dates lead to." + }, + { + "slot": "rule" + }, + { + "slot": "d1" + }, + { + "slot": "t1", + "text": "Q1" + }, + { + "slot": "e1", + "text": "Replace · milestone 1" + }, + { + "slot": "d2" + }, + { + "slot": "t2", + "text": "Q2" + }, + { + "slot": "e2", + "text": "Replace · milestone 2" + }, + { + "slot": "d3" + }, + { + "slot": "t3", + "text": "Q3" + }, + { + "slot": "e3", + "text": "Replace · milestone 3" + } + ] + }, + { + "name": "close", + "layout": "end", + "els": [ + { + "slot": "title", + "text": "Replace with the ask." + }, + { + "slot": "body", + "text": "Replace with the one next step, and who takes it." + }, + { + "slot": "caption", + "text": "PLACEHOLDER · how to reach you" + } + ] + } + ] +} diff --git a/benchmarks/authoring/reference/qbr/model.json b/benchmarks/authoring/reference/qbr/model.json new file mode 100644 index 0000000..a48f56b --- /dev/null +++ b/benchmarks/authoring/reference/qbr/model.json @@ -0,0 +1,419 @@ +{ + "$schema": "../../../../home/user/wt-bench/schemas/model.schema.json", + "title": "Meridian Freight — PLACEHOLDER deck", + "format": "slides", + "w": 960, + "h": 540, + "density": "reading", + "styles": { + "margin": 60 + }, + "master": [ + { + "id": "foot", + "footer": 1, + "right": 60, + "y": 506, + "w": "auto", + "role": "Label", + "nowrap": 1, + "text": "Meridian Freight · PLACEHOLDER" + } + ], + "slides": [ + { + "name": "cover", + "layout": "cover", + "els": [ + { + "slot": "supertitle", + "text": "MERIDIAN FREIGHT · PLACEHOLDER DECK" + }, + { + "slot": "title", + "text": "Replace with the deck's one-line promise." + }, + { + "slot": "body", + "text": "Replace with who this is for and what they get." + }, + { + "slot": "caption", + "text": "Replace · presenter, audience, date" + } + ] + }, + { + "name": "agenda", + "layout": "agenda", + "els": [ + { + "slot": "supertitle", + "text": "AGENDA" + }, + { + "slot": "title", + "text": "Replace with what the deck covers." + }, + { + "slot": "n1", + "text": "01" + }, + { + "slot": "item1", + "text": "Replace · section 1 in five words" + }, + { + "slot": "n2", + "text": "02" + }, + { + "slot": "item2", + "text": "Replace · section 2 in five words" + }, + { + "slot": "n3", + "text": "03" + }, + { + "slot": "item3", + "text": "Replace · section 3 in five words" + }, + { + "slot": "n4", + "text": "04" + }, + { + "slot": "item4", + "text": "Replace · section 4 in five words" + }, + { + "slot": "n5", + "text": "05" + }, + { + "slot": "item5", + "text": "Replace · section 5 in five words" + }, + { + "slot": "note", + "text": "Replace with the one sentence that frames the list." + } + ] + }, + { + "name": "numbers", + "template": "area-bubbles", + "fill": { + "t1": "NUMBERS", + "t2": "Replace with what the numbers say.", + "t3": "Replace with the one line the three values add up to.", + "t4": "Circle area is the value. Replace with the reading of the circles, not a repeat of the labels.", + "t5": "PLACEHOLDER · name the source", + "t6": "30", + "t7": "Replace · metric 1", + "t8": "20", + "t9": "Replace · metric 2", + "t10": "10", + "t11": "Replace · metric 3", + "data": [ + { + "label": "Replace · metric 1", + "value": 30 + }, + { + "label": "Replace · metric 2", + "value": 20 + }, + { + "label": "Replace · metric 3", + "value": 10 + } + ] + } + }, + { + "name": "trend", + "layout": "chart", + "els": [ + { + "slot": "supertitle", + "text": "TREND" + }, + { + "slot": "title", + "text": "Replace with what the series shows." + }, + { + "slot": "chart", + "chart": { + "mark": "bar", + "data": [ + { + "label": "One", + "value": 10 + }, + { + "label": "Two", + "value": 20 + }, + { + "label": "Three", + "value": 30 + }, + { + "label": "Four", + "value": 40 + } + ] + } + }, + { + "slot": "takeaway", + "text": "Replace with the takeaway. The bars above are a placeholder ramp, not data." + }, + { + "slot": "source", + "text": "PLACEHOLDER · name the source" + } + ] + }, + { + "name": "process", + "layout": "process-steps", + "textOnly": true, + "els": [ + { + "slot": "supertitle", + "text": "PROCESS" + }, + { + "slot": "title", + "text": "Replace with the name of the process." + }, + { + "slot": "n1", + "text": "01" + }, + { + "slot": "step1", + "text": "Replace · step 1, what happens and who does it" + }, + { + "slot": "n2", + "text": "02" + }, + { + "slot": "step2", + "text": "Replace · step 2, what happens and who does it" + }, + { + "slot": "n3", + "text": "03" + }, + { + "slot": "step3", + "text": "Replace · step 3, what happens and who does it" + }, + { + "slot": "n4", + "text": "04" + }, + { + "slot": "step4", + "text": "Replace · step 4, what happens and who does it" + }, + { + "slot": "note", + "text": "Replace with where the process breaks today." + } + ] + }, + { + "name": "figure", + "layout": "diagram", + "els": [ + { + "slot": "supertitle", + "text": "FIGURE" + }, + { + "slot": "title", + "text": "Replace with the name of the figure." + }, + { + "id": "a", + "x": 68, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--card)", + "bd": "1px solid var(--fg)", + "radius": 8 + }, + { + "x": 84, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace A" + }, + { + "x": 84, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "id": "b", + "x": 380, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--card)", + "bd": "1px solid var(--fg)", + "radius": 8 + }, + { + "x": 396, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace B" + }, + { + "x": 396, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "id": "c", + "x": 692, + "y": 250, + "w": 200, + "h": 60, + "bg": "var(--box)", + "bd": "2px solid var(--accent)", + "radius": 8 + }, + { + "x": 708, + "y": 258, + "w": 168, + "role": "Body", + "weight": 600, + "color": "var(--fg)", + "nowrap": 1, + "text": "Replace C" + }, + { + "x": 708, + "y": 286, + "w": 168, + "role": "Label", + "nowrap": 1, + "color": "var(--muted)", + "text": "placeholder node" + }, + { + "x": 268, + "y": 280, + "line": [ + 380, + 280 + ], + "h": 2.5, + "bg": "var(--fg)", + "from": "a", + "to": "b", + "arrow": "end", + "head": "triangle" + }, + { + "x": 580, + "y": 280, + "line": [ + 692, + 280 + ], + "h": 3, + "bg": "var(--accent)", + "from": "b", + "to": "c", + "arrow": "end", + "head": "triangle" + }, + { + "slot": "caption", + "text": "Replace with the sentence this figure proves." + } + ] + }, + { + "name": "options", + "layout": "comparison", + "textOnly": true, + "els": [ + { + "slot": "supertitle", + "text": "OPTIONS" + }, + { + "slot": "title", + "text": "Replace with the choice being made." + }, + { + "slot": "left-head", + "text": "Replace · option A" + }, + { + "slot": "right-head", + "text": "Replace · option B" + }, + { + "slot": "left", + "text": "Replace with what option A costs and buys. Two lines at most." + }, + { + "slot": "right", + "text": "Replace with what option B costs and buys. Two lines at most." + }, + { + "slot": "source", + "text": "PLACEHOLDER · name the source" + } + ] + }, + { + "name": "close", + "layout": "end", + "els": [ + { + "slot": "title", + "text": "Replace with the ask." + }, + { + "slot": "body", + "text": "Replace with the one next step, and who takes it." + }, + { + "slot": "caption", + "text": "PLACEHOLDER · how to reach you" + } + ] + } + ] +} diff --git a/benchmarks/authoring/reference/run.json b/benchmarks/authoring/reference/run.json new file mode 100644 index 0000000..b906228 --- /dev/null +++ b/benchmarks/authoring/reference/run.json @@ -0,0 +1,13 @@ +{ + "agent": "reference", + "model": "hand-built", + "evidence": false, + "note": "Wiring proof for benchmark.mjs: candidates built by hand and by bin/new.mjs. Never benchmark evidence.", + "cases": { + "pitch": {"minutes": 0, "repairs": 0}, + "qbr": {"minutes": 0, "repairs": 0}, + "one-pager": {"minutes": 0, "repairs": 1}, + "carousel": {"minutes": 0, "repairs": 0}, + "landscape": {"minutes": 0, "repairs": 0} + } +} diff --git a/benchmarks/authoring/results/.gitkeep b/benchmarks/authoring/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/llms.txt b/llms.txt index 86a0ed5..0bfe89c 100644 --- a/llms.txt +++ b/llms.txt @@ -30,6 +30,7 @@ PDF: the ⤓ button writes a true slide-sized PDF inside the file (foreignObject - docs/editor.md: the editor reference — the HUD manifest (a machine-checked list the gate compares with the template), phones, the contact sheet, both PDF routes, PNG export, presenting, persistence, the flagged-word panel and the bug door - template.html: the engine with an empty model; create.mjs fills the /*DECK*/ /*TOKENS*/ /*KEY*/ /*ENGINE*/ /*LOG*/ /*SPELL*/ markers (the /*EDITS*/ block is the in-file edit core, shared with lib/edits.mjs; the /*BUG*/ block is the bug-report builder, shared with lib/bug.mjs; #canvas opens with a