✨ Select structured data with Query and render YAML (#934)

architecture.md

@@ -448,6 +448,33 @@ durable is written.
   `undefined`, and a replay reconstructs the same validated prop set through
   ordinary expansion.
 
+### The query engine a component loads
+
+`<Query>` and `<Yaml>` select and render through one published package rather
+than through an engine this project owns. Three properties make that an
+ordinary component concern rather than a new piece of architecture.
+
+- **It is loaded by the invocation that needs it.** A literal
+  `until(import("yq.rs"))` keeps the specifier in the module graph a compiled
+  binary and a published package are built from, and makes the load an ordinary
+  operation of the element that asked: cancellation stops that element's
+  continuation, its binding and its output. The package's root export is
+  already-ready — its own top-level await compiles the embedded WebAssembly —
+  so nothing here owns an instance, and loading twice is the module cache's
+  concern. Module compilation that finishes after cancellation authorizes no
+  result: the cancelled invocation publishes nothing.
+- **Capabilities arrive as arguments, so none is given.** The evaluator reaches
+  the environment, the filesystem and the clock only through its `env`, `files`
+  and `now` options. Both call sites pass empty maps and omit `now`, which is
+  what makes an environment read an empty string, a file load a refusal and a
+  clock a refusal — and no process is ever started. A document that wants data
+  in a query hands it over as the value.
+- **Acceptance is the engine's existing strict JSON.** The evaluation's whole
+  output is parsed as one JSON value and then crosses `parseJson`, so a
+  selection joins the same value domain as everything else a document holds.
+  Zero and several results are refusals rather than guesses, and a refusal
+  binds nothing.
+
 ### Expansion identity
 
 Core describes the executable element currently being expanded:

deno.json

@@ -49,7 +49,8 @@
     "typescript": "npm:typescript@^5.0.0",
     "unist-util-select": "npm:unist-util-select@^5",
     "zod": "npm:zod@^4.3.6",
-    "mdast-util-to-string": "npm:mdast-util-to-string@^4"
+    "mdast-util-to-string": "npm:mdast-util-to-string@^4",
+    "yq.rs": "npm:yq.rs@0.1.4"
   },
   "tasks": {
     "xmd": "deno run --allow-all packages/cli/src/deno.ts",

packages/cli/tests/repl-entries.test.ts

@@ -142,6 +142,33 @@ const INHERITING = [
   "",
 ].join("\n");
 
+/**
+ * One entry that selects out of data it computed, renders it both ways, and
+ * publishes the selected value for the next entry.
+ */
+const SELECTS_AND_RENDERS = [
+  "```js eval",
+  'const report = { service: "api", checks: [{ name: "boot", ok: true }, { name: "ping", ok: false }] };',
+  "```",
+  "",
+  `<Query value={report} yq='[.checks[] | select(.ok == false) | .name]' as="failing" />`,
+  "",
+  "<Json value={failing} />",
+  "",
+  "<Yaml value={failing} />",
+  "",
+  "```js eval",
+  "const selected = failing;",
+  "const picked = selected[0];",
+  "```",
+  "",
+  "One: {picked}",
+  "",
+].join("\n");
+
+/** One entry that queries what it inherited and prints the result unbound. */
+const QUERIES_INHERITED = ['<Query value={selected} yq=".[0]" />', ""].join("\n");
+
 /** One entry that publishes a nested value a later entry will edit. */
 const PUBLISHES_NESTED = [
   "```js eval",
@@ -777,6 +804,60 @@ describe("REPL entries: what each entry owns and what the next one inherits", ()
     expect(value(after.entries[1].bindings, "seen")).toBe("draft/build/2");
   });
 
+  it("EP1: a selected value is retained, inherited and queried again", function* () {
+    const physical = new InMemoryStream();
+    const first = yield* runEntry(physical, SELECTS_AND_RENDERS);
+    expect(first.failure).toBe(undefined);
+
+    // Both renderers wrote the same selected value in their own spelling, and
+    // the entry's own output is what a person read.
+    expect(first.output).toContain('[\n  "ping"\n]');
+    expect(first.output).toContain("- ping\n");
+    expect(first.output).toContain("One: ping");
+
+    // What the entry retained is the structure, not either rendering of it —
+    // the selected binding itself is ordinary root data, beside the value it
+    // was selected from.
+    const before = projected(yield* physical.readAll());
+    expect(value(before.entries[0].bindings, "failing")).toEqual(["ping"]);
+    expect(value(before.entries[0].bindings, "selected")).toEqual(["ping"]);
+    const supplied = entryInitialBindings(before);
+    expect(supplied).toEqual({
+      report: {
+        service: "api",
+        checks: [
+          { name: "boot", ok: true },
+          { name: "ping", ok: false },
+        ],
+      },
+      failing: ["ping"],
+      selected: ["ping"],
+      picked: "ping",
+    });
+
+    // The next entry inherits that value and queries it without `as`, so what
+    // it prints is JSON — a selected string arrives quoted.
+    const second = yield* runEntry(physical, QUERIES_INHERITED, supplied);
+    expect(second.failure).toBe(undefined);
+    expect(second.output).toContain('"ping"');
+
+    // Entry 1 is where it was: its own retained value did not move, and the
+    // record handed over is unchanged.
+    const after = projected(yield* physical.readAll());
+    expect(after.entries).toHaveLength(2);
+    expect(value(after.entries[0].bindings, "selected")).toEqual(["ping"]);
+    expect(supplied.selected).toEqual(["ping"]);
+    // The second entry computed nothing of its own to publish, so what it
+    // retains is exactly the names it inherited: querying a value is not an
+    // effect with a record of its own.
+    expect(after.entries[1].bindings.map((binding) => binding.name).sort()).toEqual([
+      "failing",
+      "picked",
+      "report",
+      "selected",
+    ]);
+  });
+
   it("EP1: a retained `props` is the entry's own and reaches no successor", function* () {
     const physical = new InMemoryStream();
     const run = yield* runEntry(physical, PUBLISHES_PROPS);

packages/core/deno.json

@@ -22,6 +22,7 @@
     "zod": "npm:zod@^4.3.6",
     "unist-util-select": "npm:unist-util-select@^5",
     "mdast-util-to-string": "npm:mdast-util-to-string@^4",
-    "@secretlint/profiler": "npm:@secretlint/profiler@13.0.4"
+    "@secretlint/profiler": "npm:@secretlint/profiler@13.0.4",
+    "yq.rs": "npm:yq.rs@0.1.4"
   }
 }

packages/core/package.json

@@ -34,6 +34,7 @@
     "mdast-util-to-string": "^4",
     "remark": "15",
     "remend": "^1.2.2",
-    "unist-util-select": "^5"
+    "unist-util-select": "^5",
+    "yq.rs": "0.1.4"
   }
 }

packages/core/src/components/Query.ts

@@ -0,0 +1,98 @@
+/**
+ * `<Query>` — select one JSON value out of a value the document already has
+ * (specs/executable-mdx-spec.md §6.22).
+ *
+ * `<File>` reads text, `<Glob>` lists paths, an eval block computes data; this
+ * takes one of those values and answers with the part of it the author asked
+ * for. The expression language is `yq`'s, and the whole of the request is the
+ * value and the expression:
+ *
+ * ```xmd
+ * <Query value={packageText} yq="from_json | .scripts" as="scripts" />
+ * <Query value={scripts} yq=".test" />
+ * ```
+ *
+ * With `as` the selected value is bound as data and nothing is written. Without
+ * it the selected value is written as JSON — every type, including strings,
+ * which are quoted and escaped the way `<Json>` writes them. One spelling, one
+ * reading: an author never has to work out whether what they are looking at was
+ * a string before it was printed.
+ *
+ * ## It queries a value, and runs nothing to get one
+ *
+ * The value arrives as an ordinary prop, so whatever produced it has already
+ * finished and been bound. Querying it twice asks the same question of the same
+ * data twice: no file is read again, no request is repeated, and a paged
+ * producer's next page is not fetched because this never learns there is one.
+ *
+ * Text stays text. A JSON-looking string selected out of a value is a string,
+ * and `from_json` or `from_yaml` in the expression is how an author says to
+ * decode it — which is also what keeps a document that meant to match a quoted
+ * body from silently matching its contents instead.
+ *
+ * ## One value is the answer
+ *
+ * `null`, `false` and `[]` are answers. No answer at all and several answers
+ * are not: both refuse, and both say to collect with `[ … ]`, because an
+ * expression that yielded three results would otherwise have to pick one and an
+ * expression that yielded none would have to invent one.
+ *
+ * A refused selection binds nothing and prints nothing selected. The previous
+ * value of a binding this element would have written stays what it was — a
+ * failed query leaves the document with the data it already had rather than
+ * with the input it was handed.
+ */
+
+import type { Operation, Result } from "effection";
+import { hasBinding, hasContent } from "../component-api.ts";
+import { printErrors } from "../component-failures.ts";
+import { parseJson } from "../json.ts";
+import { select } from "./query-engine.ts";
+import type { Json } from "../types.ts";
+
+export const props = {
+  type: "object",
+  properties: {
+    value: {},
+    yq: { type: "string", minLength: 1 },
+  },
+  required: ["value", "yq"],
+  additionalProperties: false,
+};
+
+/** An invocation `<Query>` cannot run, or a selection it cannot accept. */
+export class QueryError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "QueryError";
+  }
+}
+
+const CONTENT =
+  '<Query> selects from the value it is given, not from content: write <Query value={…} yq="…" />.';
+const FAILED = "<Query> could not evaluate this expression.";
+
+export default printErrors(function* Query(props: Record<string, Json>): Operation<Json> {
+  if (yield* hasContent()) {
+    throw new QueryError(CONTENT);
+  }
+  // Asked before the selection, because the answer belongs to how the element
+  // was written rather than to what the expression found.
+  const bound = yield* hasBinding();
+  const expression = String(props.yq);
+
+  let selection: Result<string>;
+  try {
+    selection = yield* select(expression, JSON.stringify(props.value ?? null));
+  } catch (error) {
+    throw new QueryError(FAILED, { cause: error });
+  }
+  if (!selection.ok) {
+    throw new QueryError(selection.error.message, { cause: selection.error });
+  }
+
+  // The whole answer, through the engine's own strict acceptance: a binding
+  // receives the domain every other value in a document belongs to.
+  const selected = parseJson(JSON.parse(selection.value));
+  return bound ? selected : JSON.stringify(selected, null, 2);
+});

packages/core/src/components/Yaml.ts

@@ -0,0 +1,94 @@
+/**
+ * `<Yaml>` — render a structured value as YAML text
+ * (specs/executable-mdx-spec.md §6.23).
+ *
+ * `<Json>`'s sibling, for the other spelling of the same value:
+ *
+ * ```xmd
+ * <Yaml value={scripts} />
+ * ```
+ *
+ * One supplied value becomes one piece of YAML — written where the element is,
+ * or captured by `as`. The text is the query package's own rendering, returned
+ * exactly as it produced it, including the newline it ends with: a reader
+ * comparing this against `yq`'s output elsewhere is looking at the same
+ * renderer.
+ *
+ * The operand is a **capture**, for the reason `<Json>`'s is (§6.5): an
+ * ordinary prop would cross the component JSON boundary first and serialize the
+ * value before this ever saw it, which would decide the diagnostics below
+ * somewhere else.
+ *
+ * ## Shape is decided before the operand runs
+ *
+ * Content, then the syntactic presence of `value`, and only then the captured
+ * operand — so a malformed invocation costs nothing. `capture()` stays outside
+ * the `try`, because an expression that throws before it produces a value is
+ * that invocation's ordinary captured-expression failure rather than a
+ * rendering failure.
+ *
+ * ## One serialization, then one rendering
+ *
+ * `JSON.stringify` runs once, and its failures stay the two different facts
+ * they are: no JSON text at all for a value that has none, and a thrown
+ * exception for a serialization that started and could not finish. A value
+ * carrying a getter or a `toJSON` hook belongs to the document, and asking it
+ * twice would run somebody else's code twice.
+ *
+ * What the renderer does with the serialized value — key order, scalar quoting,
+ * block style, how `null` and `false` are written — is the package's, and this
+ * restates none of it. The value a selection bound stays data; rendering it
+ * here does not change it.
+ */
+
+import type { Operation } from "effection";
+import { capture, hasCapture, hasContent } from "../component-api.ts";
+import { printErrors } from "../component-failures.ts";
+import { yaml } from "./query-engine.ts";
+
+export const props = {
+  type: "object",
+  properties: {},
+  additionalProperties: false,
+};
+
+/** An invocation `<Yaml>` cannot render, or a value it cannot turn into text. */
+export class YamlRenderError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "YamlRenderError";
+  }
+}
+
+const CONTENT = "<Yaml> renders the value it is given, not content: write <Yaml value={…} />.";
+const MISSING = "<Yaml> requires a `value` prop: write <Yaml value={…} />.";
+const NO_TEXT = "<Yaml> serialization produced no JSON text for this value.";
+const THREW = "<Yaml> serialization of this value failed.";
+const RENDER = "<Yaml> could not render this value as YAML.";
+
+export default printErrors(function* Yaml(): Operation<string> {
+  if (yield* hasContent()) {
+    throw new YamlRenderError(CONTENT);
+  }
+  if (!(yield* hasCapture("value"))) {
+    throw new YamlRenderError(MISSING);
+  }
+
+  const value = yield* capture("value");
+
+  let json: unknown;
+  try {
+    json = JSON.stringify(value);
+  } catch (error) {
+    throw new YamlRenderError(THREW, { cause: error });
+  }
+  if (typeof json !== "string") {
+    throw new YamlRenderError(NO_TEXT);
+  }
+
+  try {
+    return yield* yaml(json);
+  } catch (error) {
+    throw new YamlRenderError(RENDER, { cause: error });
+  }
+});