From 0faff53c32b062b8a2f9d1b466e1486c78b73e19 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 3 Sep 2026 10:27:03 -0500 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=93=9D=20Document=20phase=20binding?= =?UTF-8?q?=20architecture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 7 + docs/binding.md | 366 +++++++++++++++++++++++++++++++++++++++++++++++ docs/insights.md | 153 ++++++++++++++++++++ 3 files changed, 526 insertions(+) create mode 100644 docs/binding.md create mode 100644 docs/insights.md diff --git a/AGENTS.md b/AGENTS.md index f8622f7..fbec01a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,13 @@ established yet. Design from the bottom up with small API sketches, exact type expectations, and tests before implementation. Do not carry architectural assumptions forward from pre-rebuild history. +## Design references + +- Read [`docs/insights.md`](docs/insights.md) before changing the architecture; + it records the current design invariants and open questions. +- Read [`docs/binding.md`](docs/binding.md) before changing tokenization, route + segmentation, phase binding, source precedence, Values, or Env handling. + ## Conventions ### File layout diff --git a/docs/binding.md b/docs/binding.md new file mode 100644 index 0000000..e7f34e0 --- /dev/null +++ b/docs/binding.md @@ -0,0 +1,366 @@ +# Phase binding + +`bindPhase()` is a source scheduler, not a source interpreter. + +CLI, Values, and Env all attempt to supply values for parameters, but they do +not traverse their inputs in the same way. CLI is positional: its safely +visible input can grow as options and arguments consume words. Values and Env +are address sources: once the route and parameter address are known, their +visibility is stable. + +The binding algorithm should preserve that distinction while giving every +source the same attempt protocol. The intended precedence is visible directly +in its structure: + +```text +CLI fixed point +Values +Env +undefined +``` + +Adding another non-positional source should require one source adapter and one +entry in the ordered source list, not another branch in `bindPhase()`. + +## Attempts and remainders + +Every immutable source remainder belongs in one object so there is exactly one +place to propagate it: + +```ts +interface Rest { + readonly tokens: Tokenizer; + readonly values: Values; + readonly envs: Envs; +} +``` + +A source attempt must distinguish absence from a captured value that failed: + +```ts +type Attempt = Maybe<{ + readonly rest: Rest; + readonly result: Result; +}>; +``` + +The outcomes mean: + +- `exists: false`: this source supplied nothing, so the next source may be + tried. +- `exists: true` and `result.ok: true`: the source supplied a valid value and + the parameter is settled. +- `exists: true` and `result.ok: false`: the source captured or supplied + something invalid. The parameter is still settled and lower-priority sources + are blocked. + +A malformed CLI option is therefore an existing, failed attempt because it +claimed input. A decoding or schema error from a higher-priority source must +not disappear by silently selecting a lower-priority value. + +Explicit `undefined` from a JavaScript value source is also present. It must be +distinguished from an absent property, must shadow lower-priority sources, and +must be passed to the schema. + +Only after every source reports absence should the schema receive `undefined`. +This final validation is where required, optional, and defaulting schemas +diverge. + +Source adapters share the attempt shape while retaining their own capture +mechanics: + +```ts +fromCLI({ param, window, rest }): Attempt; +fromValues({ param, route, rest }): Attempt; +fromEnv({ param, route, rest }): Attempt; +``` + +CLI and Env decode captured text before validating it. Values are already +JavaScript values and go directly to schema validation. + +## The CLI horizon + +An unresolved CLI word may be either parameter data or a route selector that +cannot be discovered until a later dynamic phase. The parser cannot safely let +the current route inspect the suffix until that word's role is known. + +The **horizon** is the first remaining word in the current route segment. It is +the inclusive boundary of safe lookahead. This is not a cursor: it does not +identify the current consumption position. + +The horizon itself must be visible so an option or positional argument can +prove that the word is data by claiming it. The entire suffix after it remains +hidden because, if the horizon becomes a child selector, every following +option belongs to that child. + +### Dynamic route discovery + +Consider: + +```text +index: 0 1 2 3 4 +argv: -c app.json auth0 --port 9001 + ^ + horizon +window: [--------------] |----- hidden -----| +``` + +The current phase may inspect indices 0 and 1. Its `config` parameter claims +`-c app.json`, proving that `app.json` is data. The next remaining word becomes +the horizon: + +```text +index: 0 1 2 3 4 +argv: -c app.json auth0 --port 9001 + ^ + horizon +window: [---] |--- hidden ---| +``` + +Nothing in the current phase claims `auth0`, so the horizon remains unchanged +after a complete parameter sweep. CLI binding has reached a fixed point and +must stop. + +If `resume()` introduces an `auth0` route, route search can now claim index 2 as +its selector. The child receives `--port 9001`. Without the horizon, a parent +parameter named `port` could see through the unresolved word and steal the +child's option. + +### Retrying provisional misses + +Consider: + +```text +--target local --port 9000 auth0 + ^ + horizon +``` + +Suppose `port` is visited before `target`. + +During the first sweep, `port` cannot see `--port 9000`, so its miss is only +provisional. `target` claims `--target local`, consuming the horizon. The next +remaining word is now `9000`: + +```text +--target local --port 9000 auth0 + ^ + horizon +``` + +The next sweep retries `port`, which claims `--port 9000`. The horizon then +moves to `auth0`: + +```text +--target local --port 9000 auth0 + ^ + horizon +``` + +No parameter claims it, so the horizon is stable and CLI binding stops. This +is a fixed-point calculation rather than an ordinary left-to-right cursor. + +Route search always runs before phase binding. A currently discoverable route +therefore beats an option value. A route introduced only after the current +requirement cannot retroactively preempt a word legitimately consumed by the +current phase; the parser could not yet know that route name. Setter syntax is +the unambiguous escape hatch. + +## Tokenizer support + +There is one global tokenizer and one global claim ledger for a parse. A +binding phase needs borrowed, bounded views into that tokenizer rather than +temporary tokenizers whose claims later have to be reconstructed. + +The central addition is a first-class window: + +```ts +interface TokenInput extends Iterable { + claimOne(/* ... */): Claim; + claimPair(/* ... */): Claim; + claimAll(/* ... */): Claim; +} + +class Tokenizer implements TokenInput { + window(options: { + range: TokenRange; + through?: number; + }): TokenInput; +} +``` + +A window changes visibility only. It never owns claims. Every claim made +through a window returns a remainder rooted in the global tokenizer. As a +result, `bindPhase()` can propagate the returned tokenizer directly and never +needs to: + +1. Materialize the visible tokens. +2. Compare them with a temporary remainder. +3. Recover the claimed indices. +4. Replay those claims against the global tokenizer. + +The tokenizer should materialize one stable token array and carry one +cumulative immutable set of claimed indices. Claims should not create an +ever-deepening stack of tokenizer iterators. + +The remaining invariants are: + +- Stable global token indices identify every claim. +- `through` is inclusive. +- An absent `through` means the full route segment is visible. +- Finding the first remaining word is CLI-binding policy, not generic tokenizer + policy, and can remain a small binding helper. + +Route segments are contiguous index intervals between selector tokens. The +existing segment start and end can therefore become a real range: + +```ts +interface TokenRange { + readonly start: number; // exclusive + readonly end?: number; // exclusive +} +``` + +Claimed tokens leave holes inside the interval. There is no need to copy token +arrays into each segment. + +## Target `bindPhase()` loop + +The target implementation is deliberately small: + +```ts +function bindPhase(options: BindPhaseOptions): PhaseBinding { + let { phase, segment } = options; + let rest = options.rest; + + let pending = new Map( + Object.entries(phase.params) as [string, Param][], + ); + let results = new Map>(); + + function accept( + param: Param, + attempt: Attempt, + ): void { + if (!attempt.exists) { + return; + } + + rest = attempt.value.rest; + results.set(param.name, attempt.value.result); + pending.delete(param.name); + } + + // Positional source: expand safe lookahead to a fixed point. + while (true) { + let before = firstWord(rest.tokens, segment.range); + + for (let param of pending.values()) { + let window = rest.tokens.window({ + range: segment.range, + through: before?.index, + }); + + accept(param, fromCLI({ param, window, rest })); + } + + let after = firstWord(rest.tokens, segment.range); + if (after?.index === before?.index) { + break; + } + } + + // Stable address sources: array order is precedence. + for (let source of [fromValues, fromEnv]) { + for (let param of pending.values()) { + accept(param, source({ + param, + route: segment.path, + rest, + })); + } + } + + // Absence is conclusive only after every source misses. + for (let param of pending.values()) { + results.set( + param.name, + validate(param, undefined, [param.name]), + ); + } + + return collect({ rest, results }); +} +``` + +The source-outer address loop makes precedence explicit. Deleting a parameter +from `pending` on every existing attempt prevents fallback after invalid input. +Comparing the first remaining word before and after a sweep expresses the fixed +point directly and removes `has()` and `next()` cursor bookkeeping. + +When the final word is consumed, the changed before/after comparison causes one +additional unbounded sweep. That sweep exposes trailing flags or setters. With +no remaining words before or after it, the next comparison terminates. + +## Parser integration + +A phase owns the introduction of its value and environment sources; parser +state owns their lifetime after introduction. + +When a phase becomes active, mount its newly declared sources at the current +route path: + +```ts +rest = { + ...rest, + values: rest.values.mount(segment.path, phase.values), + envs: rest.envs.mount(segment.path, phase.envs), +}; +``` + +Those sources remain available to every downstream phase. Input-level Values +and Env sources are mounted at `/` before the first phase. Sources introduced +on a child route are mounted at that child's route path. Sources introduced by +a dynamic continuation become visible only after that continuation is resumed. + +Parser state should carry `rest` as one value and replace it wholesale with the +phase result: + +```ts +state = { + ...state, + rest: binding.rest, +}; +``` + +This prevents each new source family from creating another independently +forgotten remainder assignment. Phase stitching must also concatenate source +declarations just as it merges phase parameters and routes. + +Shadowed-source diagnostics should remain a later, non-claiming inspection or +audit pass. They must not complicate or weaken winner selection in the primary +binding loop. + +## Behavioral coverage + +The design should be established with observable tests for these cases: + +- A claim through a token window updates the global tokenizer while preserving + hidden tokens. +- Consuming the horizon exposes the next word and retries previously absent + parameters. +- A stable horizon leaves the entire suffix untouched. +- CLI overrides Values even when its option is not visible until a later sweep. +- Values override Env when that is the declared source order. +- Invalid CLI blocks valid Values and Env. +- Invalid Values block valid Env. +- Only total absence invokes required, optional, or defaulting schema behavior. +- A dynamically introduced child receives every token after its selector. +- Sources introduced by a phase become available at that phase and persist + downstream. + +The essential simplification is not to pretend that CLI, Values, and Env have +identical traversal mechanics. They have identical attempt semantics, but CLI +alone has a moving visibility boundary. Keeping that distinction explicit is +what prevents source binding from becoming a collection of intertwined special +cases. diff --git a/docs/insights.md b/docs/insights.md new file mode 100644 index 0000000..35c9862 --- /dev/null +++ b/docs/insights.md @@ -0,0 +1,153 @@ +# Configliere invariants + +These principles are drawn from the current rebuild and earlier design +transcripts. Current decisions take precedence over superseded architectures. +`★` marks principles that received especially enthusiastic assent. + +The detailed phase-binding design is recorded in [Binding](./binding.md). + +## Design + +- ★ Sketch the ideal API first; write exact type expectations second; test + third; implement last. +- ★ Hover types are part of the product. +- The primary generic represents the actual runtime value, not a definition tree + or helper-type calculation. +- Definitions should read like configuration, not "config balls." +- Definitions are immutable function-composition pipelines. +- Any valid `A → B` transformation may participate in a pipeline. +- `extend()` packages context-free composition; `extend()` is identity. +- Static definitions contain enough metadata to render help and schemas without + parsing input. +- Standard Schema validates values; it does not define source syntax or token + grammar. +- Complexity may exist behind overloads and implementation-boundary casts, but + not in user-facing types. + +## Routing + +- ★ Resolve the route before interpreting remaining tokens as parameters. +- The program name is identity, not part of its route: `/`, `/serve`, + `/database/clean`. +- Command literals are routing tokens, not positional arguments. +- Every route is a distinct application entry point. +- Every route supports `HELP`. +- `VERSION` and `EXECUTE` exist only where declared. +- `command()` is exactly an executable `route()`. +- An executable route may still contain child routes. +- Controls are semantic methods, not special tokenizer token types. +- The deepest discoverable route owns the requested method. +- Control placement does not change the route: `app --help child` and + `app child --help` target the child. +- Nothing after `--` participates in routing, controls, or option binding. +- There is always at least the root route; unmatched words become binding + overflow rather than "route not found." +- Providers declare local names; mounting supplies route, namespace, and option + prefixes. + +## Input and claims + +- Tokenization describes syntax only; bindings assign semantic meaning. +- Input is immutable. +- Tokens retain stable global indices and original order. +- Parents establish scope; children claim only within that scope. +- Available input flows down; claims, issues, and models flow up. +- Claims plus remainder conserve the original input: nothing disappears or is + claimed twice. +- Unknown tokens are diagnosed only after all applicable bindings have had an + opportunity to claim them. +- The original CLI input is supplied once; continuations retain its unconsumed + portion. + +## Parameters and sources + +- ★ A parameter introduces an address and a target value type. +- A binding reads a source's physical representation. +- A decoder interprets a captured representation. +- A schema validates the interpreted model value. +- ★ Binding failure means the source representation could not be captured. +- ★ Decoding failure means it was captured but could not be interpreted. +- ★ Schema failure means it was interpreted but is not a valid model value. +- CLI, environment, and JavaScript values have source-specific readers. +- CLI and environment readers may share decoders without sharing capture + semantics. +- `switch()` owns boolean CLI grammar; schemas are never probed to infer token + behavior. +- Source selection follows explicit precedence. +- A valid higher-priority winner is not invalidated by a bad shadowed source. +- ★ Shadowed invalid sources should remain available as diagnostics. +- A child configuration is validated through the child's value binding, not + round-tripped through argv. + +## Models + +- The resolved route has a directly typed `model`. +- The result also carries path-addressed `models` for every matched route. +- Route identity and result projection are separate concerns. +- Namespacing is a mounting/address operation, not something plugin authors + repeat locally. +- Exact route and method values discriminate the result union. + +## Dynamic phases + +- ★ Dynamic parsing is a typed pause, not baked-in discovery or asynchronous + loading. +- Library parsing remains synchronous and pure; the application performs I/O + between increments. +- Every route has at least one phase. +- Parameters belong to phases and are bound phase by phase. +- Phase parameter types and increment models are phase-local slices. +- A caller has already observed earlier increments; the final intent exposes the + aggregate model. +- Once a phase is resolved, its values are immutable. +- Newly supplied values or environment sources apply only to future phases. +- A dynamic resolver returns an extension at its current pipeline position. +- The extension's return type determines the continuation type. +- `resume()` continues parsing; it never exposes the intermediate route. +- `resume()` accepts `Result`, allowing loader failures through the + ordinary issue path. +- A failed requirement never invokes the resolver. +- `RequirementsOf` preserves requirement order; `RequirementOf` is its + head. +- `ContinuationOf` removes the current phase. +- The static result type says whether parsing yields another increment or an + intent; callers should not need a runtime `done` check. +- Help and version may require resolving earlier phases when those phases can + introduce the target route. + +## Help and diagnostics + +- ★ Help should feel free at every route depth. +- `printHelp()` accepts a Help intent and returns a string; it performs no I/O. +- `printVersion()` likewise renders rather than writes. +- Help, version, and execute share route resolution. +- Unsupported methods produce method-not-allowed, not malformed pseudo-intents. +- Issues should share one rich, normalized shape compatible with Standard Schema + issues. +- Help may expose current values, winning sources, and shadowed invalid sources. +- Help intent normally precedes ordinary argument validation, except for phases + required to discover the requested route. + +## Tests + +- ★ Behavioral tests should describe public parses, not internal composition + machinery. +- Type tests exist where inference itself is the behavior. +- CLI-string fixtures and matchers such as `toHaveRoute()` and `toHaveModels()` + should make intent visible. +- Do not test private search segments directly. +- Avoid loops and helper indirection when two literal assertions communicate the + contract better. + +## Still open + +These have not been promoted to invariants: + +- Exact shadowed-source warning policy. +- Holistic model transforms. +- Final alias syntax. +- Recursive-dynamic implementation details. +- The exact route, path, and path-addressed model context exposed by an + increment. +- Recursive result typing for increments reached through child routes. +- Every edge of strict versus permissive option placement. From 22bc5657d42eb523de08518e5fcb88ac414c6531 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 3 Sep 2026 10:31:13 -0500 Subject: [PATCH 2/6] =?UTF-8?q?=E2=9C=A8=20Add=20mounted=20value=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/values.ts | 125 +++++++++++++++++++++++ test/value.test.ts | 245 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 370 insertions(+) create mode 100644 lib/values.ts create mode 100644 test/value.test.ts diff --git a/lib/values.ts b/lib/values.ts new file mode 100644 index 0000000..bf6b789 --- /dev/null +++ b/lib/values.ts @@ -0,0 +1,125 @@ +import type { Maybe } from "./maybe.ts"; +import type { AnyRoute, RoutePath } from "./types.ts"; + +export type ValueSource = { + name: string; + value: unknown; +}; + +export function withValues( + values: ValueSource[], +): (route: R) => R { + return (route) => { + let phases = [...route.phases]; + let phase = phases.pop()!; + phases.push({ + ...phase, + values: phase.values.concat(values), + }) + return { + ...route, + phases, + }; + }; +} + +export type ValueClaim = { + result: Maybe<{ + source: string; + address: string[]; + value: unknown; + }>; + rest: Values; +}; + +export class Values { + mounts: Map; + claims: Set; + + constructor(mounts: Map = new Map(), claims: Set = new Set()) { + this.mounts = mounts; + this.claims = claims; + } + + mount(path: string[], sources: ValueSource[]): Values { + if (sources.length === 0) { + return this; + } + const id = routeId(path); + + return new Values( + new Map([...this.mounts.entries(), [ + id, + (this.mounts.get(id) ?? []).concat(sources), + ]]), + this.claims, + ); + } + + claim({ route, address }: ClaimOptions): ValueClaim { + let id = claimId([...route, ...address]); + const nope: ValueClaim = { result: { exists: false }, rest: this }; + if (this.claims.has(id)) { + return nope; + } + + for (let end = route.length; end >= 0; end--) { + let mountId = routeId(route.slice(0, end)); + let sources = this.mounts.get(mountId) ?? []; + for (let { name, value } of sources) { + let result = find(value, [...route.slice(end), ...address]); + if (result.exists) { + let rest = new Values(this.mounts, new Set([...this.claims, id])); + return { + result: { + exists: true, + value: { + source: name, + address, + value: result.value, + }, + }, + rest, + }; + } + } + } + return nope; + } +} + +export interface ClaimOptions { + route: string[]; + address: string[]; +} + +type ClaimId = string; + +function routeId(address: string[]): RoutePath { + return `/${address.join("/")}`; +} + +function claimId(address: string[]): ClaimId { + return JSON.stringify(address); +} + +function find(value: unknown, path: string[]): Maybe { + let current: unknown = value; + + for (let key of path) { + if ( + current === null || + (typeof current !== "object" && typeof current !== "function") || + !Object.hasOwn(current, key) + ) { + return { exists: false }; + } + + current = (current as Record)[key]; + } + + return { + exists: true, + value: current, + }; +} diff --git a/test/value.test.ts b/test/value.test.ts new file mode 100644 index 0000000..a61e156 --- /dev/null +++ b/test/value.test.ts @@ -0,0 +1,245 @@ +import { expect } from "@std/expect"; +import { describe, it } from "@std/testing/bdd"; +import { type } from "arktype"; +import { command } from "../lib/command.ts"; +import { name } from "../lib/definition.ts"; +import { dynamic } from "../lib/dynamic.ts"; +import { extend } from "../lib/extend.ts"; +import { option } from "../lib/option.ts"; +import { schema } from "../lib/param.ts"; +import { parse } from "../lib/parse.ts"; +import { routes } from "../lib/route.ts"; +import type { Parse } from "../lib/types.ts"; + +describe("value sources", () => { + it.skip("binds top-level properties to root parameters", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + option(name("domain"), schema(type("string"))), + ); + let input = { + argv: [], + values: [{ + name: "settings", + value: { + port: 9001, + domain: "auth0.local", + }, + }], + }; + let result = parse(app, input); + + expectType>>(true); + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { + port: 9001, + domain: "auth0.local", + }, + models: { + "/": { + port: 9001, + domain: "auth0.local", + }, + }, + }); + }); + + it.skip("scopes nested properties by route name", () => { + let auth0 = command( + name("auth0"), + option(name("delay"), schema(type("number"))), + option(name("port"), schema(type("number"))), + ); + let app = command( + name("simulacrum"), + option(name("delay"), schema(type("number"))), + routes(auth0), + ); + let input = { + argv: ["auth0"], + values: [{ + name: "settings", + value: { + delay: 1000, + auth0: { + delay: 2000, + port: 9001, + }, + }, + }], + }; + let result = parse(app, input); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/auth0", + model: { + delay: 2000, + port: 9001, + }, + models: { + "/": { delay: 1000 }, + "/auth0": { + delay: 2000, + port: 9001, + }, + }, + }); + }); + + it.skip("combines disjoint properties from several value sources", () => { + let app = command( + name("simulacrum"), + option(name("host"), schema(type("string"))), + option(name("port"), schema(type("number"))), + ); + let input = { + argv: [], + values: [ + { + name: "defaults", + value: { host: "localhost" }, + }, + { + name: "settings", + value: { port: 9001 }, + }, + ], + }; + let result = parse(app, input); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { + host: "localhost", + port: 9001, + }, + }); + }); + + it.skip("validates values without applying the CLI decoder", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + ); + let input = { + argv: [], + values: [{ + name: "settings", + value: { port: "9001" }, + }], + }; + let result = parse(app, input); + + expect(result).toMatchObject({ + ok: false, + code: "unprocessable-content", + route: "/", + issues: [{ path: ["port"] }], + }); + }); + + it.skip("keeps values available to parameters introduced by a later phase", () => { + let app = command( + name("simulacrum"), + dynamic((_plugins: Plugins) => + extend( + option(name("domain"), schema(type("string"))), + ) + ), + ); + let input = { + argv: [], + values: [{ + name: "settings", + value: { domain: "auth0.local" }, + }], + }; + let increment = parse(app, input); + + assertIncrement(increment); + let result = increment.resume({ + ok: true, + value: { names: ["auth0"] }, + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { domain: "auth0.local" }, + models: { + "/": { domain: "auth0.local" }, + }, + }); + }); + + it.skip("handles a value key shared by a parameter and child route", () => { + let auth0 = command( + name("auth0"), + option(name("port"), schema(type("number"))), + ); + let app = command( + name("simulacrum"), + option(name("auth0")), + routes(auth0), + ); + let input = { + argv: ["auth0"], + values: [{ + name: "settings", + value: { + auth0: { port: 9001 }, + }, + }], + }; + let result = parse(app, input); + + // Whether the root parameter also receives `value.auth0` remains open. + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/auth0", + models: { + "/auth0": { port: 9001 }, + }, + }); + }); +}); + +interface Plugins { + readonly names: readonly string[]; +} + +type Equal = (() => T extends L ? 1 : 2) extends + (() => T extends R ? 1 : 2) + ? (() => T extends R ? 1 : 2) extends (() => T extends L ? 1 : 2) ? true + : false + : false; + +function assertIncrement( + result: T, +): asserts result is Extract< + T, + { readonly ok: true; readonly resume: unknown } +> { + expect(result).toMatchObject({ + ok: true, + route: "/", + model: {}, + }); + expect( + typeof (result as { readonly resume?: unknown }).resume, + ).toBe("function"); +} + +function expectType(_value: T): void { + // Compile-time assertion. +} From d5aedeaeea903c1f22444e1c70cb4f3473950174 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 3 Sep 2026 11:12:36 -0500 Subject: [PATCH 3/6] =?UTF-8?q?=E2=9C=A8=20Add=20tokenizer=20views=20over?= =?UTF-8?q?=20a=20shared=20token=20pool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/binding.md | 20 +++--- lib/bind.ts | 4 +- lib/param.ts | 4 +- lib/read.ts | 13 ++-- lib/tokenizer.ts | 141 +++++++++++++++++++++++++++++++++++++---- test/tokenizer.test.ts | 113 +++++++++++++++++++++++++++++++++ 6 files changed, 260 insertions(+), 35 deletions(-) diff --git a/docs/binding.md b/docs/binding.md index e7f34e0..0c5d772 100644 --- a/docs/binding.md +++ b/docs/binding.md @@ -70,7 +70,7 @@ Source adapters share the attempt shape while retaining their own capture mechanics: ```ts -fromCLI({ param, window, rest }): Attempt; +fromCLI({ param, view, rest }): Attempt; fromValues({ param, route, rest }): Attempt; fromEnv({ param, route, rest }): Attempt; ``` @@ -102,7 +102,7 @@ index: 0 1 2 3 4 argv: -c app.json auth0 --port 9001 ^ horizon -window: [--------------] |----- hidden -----| +view: [--------------] |----- hidden -----| ``` The current phase may inspect indices 0 and 1. Its `config` parameter claims @@ -114,7 +114,7 @@ index: 0 1 2 3 4 argv: -c app.json auth0 --port 9001 ^ horizon -window: [---] |--- hidden ---| +view: [---] |--- hidden ---| ``` Nothing in the current phase claims `auth0`, so the horizon remains unchanged @@ -172,7 +172,7 @@ There is one global tokenizer and one global claim ledger for a parse. A binding phase needs borrowed, bounded views into that tokenizer rather than temporary tokenizers whose claims later have to be reconstructed. -The central addition is a first-class window: +The central addition is a first-class view: ```ts interface TokenInput extends Iterable { @@ -182,15 +182,15 @@ interface TokenInput extends Iterable { } class Tokenizer implements TokenInput { - window(options: { + view(options: { range: TokenRange; through?: number; }): TokenInput; } ``` -A window changes visibility only. It never owns claims. Every claim made -through a window returns a remainder rooted in the global tokenizer. As a +A view changes visibility only. It never owns claims. Every claim made +through a view returns a remainder rooted in the global tokenizer. As a result, `bindPhase()` can propagate the returned tokenizer directly and never needs to: @@ -256,12 +256,12 @@ function bindPhase(options: BindPhaseOptions): PhaseBinding { let before = firstWord(rest.tokens, segment.range); for (let param of pending.values()) { - let window = rest.tokens.window({ + let view = rest.tokens.view({ range: segment.range, through: before?.index, }); - accept(param, fromCLI({ param, window, rest })); + accept(param, fromCLI({ param, view, rest })); } let after = firstWord(rest.tokens, segment.range); @@ -345,7 +345,7 @@ binding loop. The design should be established with observable tests for these cases: -- A claim through a token window updates the global tokenizer while preserving +- A claim through a token view updates the global tokenizer while preserving hidden tokens. - Consuming the horizon exposes the next word and retries previously absent parameters. diff --git a/lib/bind.ts b/lib/bind.ts index a9bdba2..5d1fcc3 100644 --- a/lib/bind.ts +++ b/lib/bind.ts @@ -2,7 +2,7 @@ import type { Param } from "./param.ts"; import type { Symbol } from "./read.ts"; import type { Result } from "./result.ts"; import type { Flag, Setter, Word } from "./tokenize.ts"; -import { Tokenizer } from "./tokenizer.ts"; +import { type TokenInput, Tokenizer } from "./tokenizer.ts"; import type { AnyPhase, Issue } from "./types.ts"; export interface Binding { @@ -25,7 +25,7 @@ export interface PhaseSegment { export function bind>(options: { param: P; - tokens: Tokenizer; + tokens: TokenInput; }): Binding { let { param, tokens } = options; let path = [param.name]; diff --git a/lib/param.ts b/lib/param.ts index 070d466..fd84672 100644 --- a/lib/param.ts +++ b/lib/param.ts @@ -2,7 +2,7 @@ import { type Decoder, scalar } from "./decode.ts"; import { extend } from "./extend.ts"; import type { ReadCLI } from "./read.ts"; import type { AnyToken } from "./tokenize.ts"; -import type { Tokenizer } from "./tokenizer.ts"; +import type { TokenInput } from "./tokenizer.ts"; import type { Definition, Schema } from "./types.ts"; export interface Param extends Definition { @@ -133,7 +133,7 @@ export function param( let zero = { ...start, schema: unknown, - cli: (tokens: Tokenizer) => ({ + cli: (tokens: TokenInput) => ({ result: { ok: true, value: { exists: false }, diff --git a/lib/read.ts b/lib/read.ts index 45ea77b..1dbc42e 100644 --- a/lib/read.ts +++ b/lib/read.ts @@ -2,12 +2,12 @@ import type { Maybe } from "./maybe.ts"; import type { Param } from "./param.ts"; import type { Result } from "./result.ts"; import type { Flag, Setter, Word } from "./tokenize.ts"; -import type { Claim, Tokenizer } from "./tokenizer.ts"; +import type { Claim, TokenInput } from "./tokenizer.ts"; export type Symbol = Flag | Setter | Word; export type ReadCLI = ( - tokens: Tokenizer, + tokens: TokenInput, ) => CLIRead; export interface CLIRead { @@ -99,12 +99,11 @@ export function cli( }); } -function nothing(tokenizer: Tokenizer): CLIRead { +function nothing(tokenizer: TokenInput): CLIRead { + let claim = tokenizer.claimAll(() => false); + return { - claim: { - tokens: [], - rest: tokenizer, - }, + claim, result: { ok: true, value: { exists: false }, diff --git a/lib/tokenizer.ts b/lib/tokenizer.ts index 39252d7..9469e0e 100644 --- a/lib/tokenizer.ts +++ b/lib/tokenizer.ts @@ -1,29 +1,51 @@ import type { AnyToken } from "./tokenize.ts"; -export class Tokenizer implements Iterable { - tokens: Iterable; - claimed: Set; +export interface TokenInput extends Iterable { + claimNext(): Claim; + + claimOne(match: (token: T) => token is S): Claim; + claimOne(match: (token: T) => boolean): Claim; + + claimPair(match: (a: T, b: T) => boolean): Claim; + + claimAll(match: (token: T) => token is S): Claim; + claimAll(match: (token: T) => boolean): Claim; +} + +export interface TokenRange { + readonly start: number; + readonly end?: number; +} + +export interface ViewOptions { + readonly range: TokenRange; + readonly through?: number; +} + +export class Tokenizer implements TokenInput { + readonly tokens: readonly T[]; + readonly claimed: ReadonlySet; constructor( - tokens: typeof this.tokens, - claimed: typeof this.claimed = new Set(), + tokens: readonly T[], + claimed: ReadonlySet = new Set(), ) { this.tokens = tokens; - this.claimed = claimed; + this.claimed = new Set(claimed); } claimNext(): Claim { - return this.claimOne(() => true); + return this.claimOne(() => true); } claimOne(match: (token: T) => token is S): Claim; - claimOne(match: (token: T) => boolean): Claim; + claimOne(match: (token: T) => boolean): Claim; claimOne(match: (token: T) => boolean): Claim { for (let token of this) { if (match(token)) { return { tokens: [token], - rest: new Tokenizer(this, new Set([token.index])), + rest: remainder(this, [token.index]), }; } } @@ -40,7 +62,7 @@ export class Tokenizer implements Iterable { if (match(previous, token)) { return { tokens: [previous, token], - rest: new Tokenizer(this, new Set([previous.index, token.index])), + rest: remainder(this, [previous.index, token.index]), }; } previous = token; @@ -52,15 +74,21 @@ export class Tokenizer implements Iterable { claimAll(match: (token: T) => boolean): Claim; claimAll(match: (token: T) => boolean): Claim { let tokens: T[] = []; - let claims = new Set(); + let claimed = new Set(); for (let token of this) { if (match(token)) { - claims.add(token.index); + claimed.add(token.index); tokens.push(token); } } - let rest = new Tokenizer(this, claims); - return { tokens, rest }; + return { + tokens, + rest: tokens.length > 0 ? remainder(this, claimed) : this, + }; + } + + view(options: ViewOptions): TokenInput { + return new View(this, options); } *[Symbol.iterator](): Generator { @@ -76,3 +104,88 @@ export interface Claim { tokens: T[]; rest: Tokenizer; } + +class View implements TokenInput { + constructor( + readonly source: Tokenizer, + readonly options: ViewOptions, + ) {} + + claimNext(): Claim { + return this.claimOne(() => true); + } + + claimOne(match: (token: T) => token is S): Claim; + claimOne(match: (token: T) => boolean): Claim; + claimOne(match: (token: T) => boolean): Claim { + for (let token of this) { + if (match(token)) { + return { + tokens: [token], + rest: remainder(this.source, [token.index]), + }; + } + } + return { tokens: [], rest: this.source }; + } + + claimPair(match: (a: T, b: T) => boolean): Claim { + let previous: T | undefined; + for (let token of this) { + if (!previous) { + previous = token; + continue; + } + if (match(previous, token)) { + return { + tokens: [previous, token], + rest: remainder(this.source, [previous.index, token.index]), + }; + } + previous = token; + } + return { tokens: [], rest: this.source }; + } + + claimAll(match: (token: T) => token is S): Claim; + claimAll(match: (token: T) => boolean): Claim; + claimAll(match: (token: T) => boolean): Claim { + let tokens: T[] = []; + let claimed = new Set(); + for (let token of this) { + if (match(token)) { + claimed.add(token.index); + tokens.push(token); + } + } + return { + tokens, + rest: tokens.length > 0 ? remainder(this.source, claimed) : this.source, + }; + } + + *[Symbol.iterator](): Generator { + let { start, end } = this.options.range; + let { through } = this.options; + + for (let token of this.source) { + if ( + token.index > start && + (end === undefined || token.index < end) && + (through === undefined || token.index <= through) + ) { + yield token; + } + } + } +} + +function remainder( + source: Tokenizer, + claimed: Iterable, +): Tokenizer { + return new Tokenizer( + source.tokens, + new Set([...source.claimed, ...claimed]), + ); +} diff --git a/test/tokenizer.test.ts b/test/tokenizer.test.ts index d92007c..ac2a457 100644 --- a/test/tokenizer.test.ts +++ b/test/tokenizer.test.ts @@ -42,6 +42,29 @@ describe("Tokenizer", () => { expect(indices(words.rest)).toEqual([]); }); + it("shares one token pool across independent remainders", () => { + let source = tokenize(["-h", "database", "--port=9000"]); + let tokenizer = new Tokenizer(source); + + let help = tokenizer.claimOne((token) => token.index === 0).rest; + let database = tokenizer.claimOne((token) => token.index === 1).rest; + + expect(help.tokens).toBe(tokenizer.tokens); + expect(database.tokens).toBe(tokenizer.tokens); + expect(indices(help)).toEqual([1, 2]); + expect(indices(database)).toEqual([0, 2]); + }); + + it("accumulates claims without wrapping the previous remainder", () => { + let tokenizer = new Tokenizer(tokenize(["-h", "database", "--verbose"])); + let help = tokenizer.claimOne((token) => token.index === 0).rest; + let verbose = help.claimOne((token) => token.index === 2).rest; + + expect(verbose.tokens).toBe(tokenizer.tokens); + expect(verbose.claimed).toEqual(new Set([0, 2])); + expect(indices(verbose)).toEqual([1]); + }); + it("does not test already claimed tokens again", () => { let source = tokenize(["-h", "database", "--port=9000"]); let tokenizer = new Tokenizer(source); @@ -55,6 +78,96 @@ describe("Tokenizer", () => { expect(tested).toEqual([1, 2]); }); + + describe("view()", () => { + it("shows only tokens in its range through its inclusive horizon", () => { + let tokenizer = new Tokenizer(tokenize([ + "before", + "--target", + "local", + "--port", + "9000", + "after", + ])); + let view = tokenizer.view({ + range: { start: 0, end: 5 }, + through: 2, + }); + + expect(indices(view)).toEqual([1, 2]); + }); + + it("shows the full range when there is no horizon", () => { + let tokenizer = new Tokenizer(tokenize([ + "before", + "--target", + "local", + "--port", + "9000", + "after", + ])); + let view = tokenizer.view({ + range: { start: 0, end: 5 }, + }); + + expect(indices(view)).toEqual([1, 2, 3, 4]); + }); + + it("returns globally rooted remainders from claims through a view", () => { + let tokenizer = new Tokenizer(tokenize([ + "before", + "--target", + "local", + "--port", + "9000", + "after", + ])); + let view = tokenizer.view({ + range: { start: 0, end: 5 }, + through: 2, + }); + let claim = view.claimPair((name, value) => { + return name.text === "--target" && value.text === "local"; + }); + + expect(indices(claim.tokens)).toEqual([1, 2]); + expect(claim.rest.tokens).toBe(tokenizer.tokens); + expect(indices(claim.rest)).toEqual([0, 3, 4, 5]); + }); + + it("returns the global tokenizer when a claim through a view misses", () => { + let tokenizer = new Tokenizer(tokenize([ + "before", + "--target", + "local", + "after", + ])); + let view = tokenizer.view({ + range: { start: 0, end: 3 }, + through: 2, + }); + let claim = view.claimOne((token) => token.text === "--missing"); + + expect(claim.tokens).toEqual([]); + expect(claim.rest).toBe(tokenizer); + }); + + it("honors claims already made in the global tokenizer", () => { + let tokenizer = new Tokenizer(tokenize([ + "before", + "--target", + "local", + "after", + ])); + let claimed = tokenizer.claimOne((token) => token.index === 1).rest; + let view = claimed.view({ + range: { start: 0, end: 3 }, + through: 2, + }); + + expect(indices(view)).toEqual([2]); + }); + }); }); function indices(tokens: Iterable): number[] { From 6639daafd7196ceb41d2db6bb4da1faddff1cdac Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 3 Sep 2026 12:02:57 -0500 Subject: [PATCH 4/6] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Thread=20parser=20sour?= =?UTF-8?q?ces=20through=20Rest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/bind.ts | 76 ++++++++++++++++++++---------------------- lib/dynamic.ts | 1 + lib/parse.ts | 46 +++++++++++++++++++++----- lib/rest.ts | 8 +++++ lib/route.ts | 1 + lib/types.ts | 5 +++ lib/values.ts | 23 +++++++------ mod.ts | 5 +++ test/bind.test.ts | 84 ++++++++++++++++++++++++++++++++++++----------- 9 files changed, 170 insertions(+), 79 deletions(-) create mode 100644 lib/rest.ts diff --git a/lib/bind.ts b/lib/bind.ts index 5d1fcc3..ea621de 100644 --- a/lib/bind.ts +++ b/lib/bind.ts @@ -1,17 +1,18 @@ import type { Param } from "./param.ts"; import type { Symbol } from "./read.ts"; +import type { Rest } from "./rest.ts"; import type { Result } from "./result.ts"; import type { Flag, Setter, Word } from "./tokenize.ts"; -import { type TokenInput, Tokenizer } from "./tokenizer.ts"; +import type { TokenInput, Tokenizer, TokenRange } from "./tokenizer.ts"; import type { AnyPhase, Issue } from "./types.ts"; export interface Binding { - rest: Tokenizer; + rest: Rest; result: Result; } export interface PhaseBinding { - rest: Tokenizer; + rest: Rest; model: Record; issues: Issue[]; valid: boolean; @@ -19,17 +20,22 @@ export interface PhaseBinding { } export interface PhaseSegment { - readonly tokens: Iterable; + readonly range: TokenRange; readonly cursor?: number; } export function bind>(options: { param: P; - tokens: TokenInput; + view: TokenInput; + rest: Rest; }): Binding { - let { param, tokens } = options; + let { param, view, rest } = options; let path = [param.name]; - let cli = param.cli(tokens); + let cli = param.cli(view); + let next = { + ...rest, + tokens: cli.claim.rest, + }; if (cli.result.ok) { let read = cli.result.value; @@ -39,7 +45,7 @@ export function bind>(options: { if (candidates.length === 0) { return { - rest: cli.claim.rest, + rest: next, result: { ok: false, issues: [{ @@ -55,7 +61,7 @@ export function bind>(options: { let result = validate(param, candidate, path); if (result.ok) { return { - rest: cli.claim.rest, + rest: next, result, }; } else { @@ -64,7 +70,7 @@ export function bind>(options: { } return { - rest: cli.claim.rest, + rest: next, result: { ok: false, issues: issues ?? [], @@ -73,7 +79,7 @@ export function bind>(options: { } return { - rest: cli.claim.rest, + rest: next, result: { ok: false, issues: cli.result.issues, @@ -84,12 +90,11 @@ export function bind>(options: { export function bindPhase(options: { phase: AnyPhase; segment: PhaseSegment; - tokens: Tokenizer; + rest: Rest; }): PhaseBinding { let { phase, segment } = options; - let tokens = options.tokens; + let rest = options.rest; let params = Object.values(phase.params) as Param[]; - let members = new Set(Array.from(segment.tokens, (token) => token.index)); let cursor = segment.cursor; let results = new Map>(); let settled = new Set(); @@ -100,36 +105,29 @@ export function bindPhase(options: { continue; } - let visible = Array.from(tokens).filter((token) => { - return members.has(token.index) && - (cursor === undefined || token.index <= cursor); - }); + let tokens = rest.tokens; let binding = bind({ param, - tokens: new Tokenizer(visible), + view: tokens.view({ + range: segment.range, + through: cursor, + }), + rest, }); - let remaining = new Set( - Array.from(binding.rest, (token) => token.index), - ); - let claimed = new Set( - visible - .filter((token) => !remaining.has(token.index)) - .map((token) => token.index), - ); results.set(param.name, binding.result); + rest = binding.rest; - if (claimed.size > 0) { - tokens = tokens.claimAll((token) => claimed.has(token.index)).rest; + if (rest.tokens !== tokens) { settled.add(param.name); } } - if (cursor === undefined || has(tokens, members, cursor)) { + if (cursor === undefined || has(rest.tokens, segment.range, cursor)) { break; } - cursor = next(tokens, members, cursor); + cursor = next(rest.tokens, segment.range, cursor); } let model: Record = {}; @@ -151,7 +149,7 @@ export function bindPhase(options: { } } - return { rest: tokens, model, issues, valid, cursor }; + return { rest, model, issues, valid, cursor }; } function validate>( @@ -190,11 +188,11 @@ function validate>( function has( tokens: Tokenizer, - members: Set, + range: TokenRange, index: number, ): boolean { - for (let token of tokens) { - if (members.has(token.index) && token.index === index) { + for (let token of tokens.view({ range })) { + if (token.index === index) { return true; } } @@ -203,13 +201,11 @@ function has( function next( tokens: Tokenizer, - members: Set, + range: TokenRange, index: number, ): number | undefined { - for (let token of tokens) { - if ( - members.has(token.index) && token.index > index && token.type === "word" - ) { + for (let token of tokens.view({ range })) { + if (token.index > index && token.type === "word") { return token.index; } } diff --git a/lib/dynamic.ts b/lib/dynamic.ts index d3822ab..7244f75 100644 --- a/lib/dynamic.ts +++ b/lib/dynamic.ts @@ -27,6 +27,7 @@ export function dynamic< phases.push({ params: {}, routes: [], + values: [], }); return { ...route, phases } as unknown as Conjoin< diff --git a/lib/parse.ts b/lib/parse.ts index 7aaf3e7..0da56f5 100644 --- a/lib/parse.ts +++ b/lib/parse.ts @@ -1,8 +1,10 @@ import { bindPhase } from "./bind.ts"; import type { Symbol } from "./read.ts"; +import type { Rest } from "./rest.ts"; import type { Result } from "./result.ts"; import { type AnyToken, type Literal, tokenize } from "./tokenize.ts"; import { Tokenizer } from "./tokenizer.ts"; +import { Values } from "./values.ts"; import type { AnyIntent, AnyPhase, @@ -42,14 +44,17 @@ export function parse( method = "version"; } - let rest = literals.rest as Tokenizer; + let rest: Rest = { + tokens: literals.rest as Tokenizer, + values: new Values().mount([], input.values ?? []), + }; return resume({ segments: [{ id: "/", route, phases: route.phases, - tokens: Array.from(rest), + tokens: Array.from(rest.tokens), path: [], start: -1, model: {}, @@ -75,6 +80,14 @@ function resume( let segment = state.segments[index]; let [phase] = segment.phases; + state = { + ...state, + rest: { + ...state.rest, + values: state.rest.values.mount(segment.path, phase.values), + }, + }; + // Controls are free of ordinary configuration validation. A dynamic phase // is the exception: its model is needed by the caller before the remaining // route graph (and therefore the final control target) can be known. @@ -97,14 +110,20 @@ function resume( return resolve({ ...state, models }, segment); } - // Bind only this phase, within this segment's token window. + // Bind only this phase, within this segment's token view. // // For an open dynamic segment, a binding may consume the cursor but may // not leap over it. For a committed segment, `end` is the hard boundary. let binding = bindPhase({ phase, - segment, - tokens: state.rest, + segment: { + range: { + start: segment.start, + end: segment.end, + }, + cursor: segment.cursor, + }, + rest: state.rest, }); segment = { @@ -228,7 +247,7 @@ function resume( interface ParserState { segments: [Segment, ...Segment[]]; active: number; - rest: Tokenizer; + rest: Rest; models: ModelsByRoute; method: Method; literals: Literal[]; @@ -263,7 +282,7 @@ function search(state: ParserState): ParserState { let index = segments.length - 1; let segment = segments[index]; let [phase] = segment.phases; - let remaining = new Set(Array.from(rest, (token) => token.index)); + let remaining = new Set(Array.from(rest.tokens, (token) => token.index)); let selector = segment.tokens.find((token) => { return remaining.has(token.index) && token.type === "word" && phase.routes.some((route) => route.name === token.text); @@ -301,7 +320,11 @@ function search(state: ParserState): ParserState { history: [], }); - rest = rest.claimOne((token) => token.index === selector.index).rest; + rest = { + ...rest, + tokens: rest.tokens.claimOne((token) => token.index === selector.index) + .rest, + }; } } @@ -332,6 +355,10 @@ function stitch( ...phase.routes, ...next.routes, ], + values: [ + ...phase.values, + ...next.values, + ], }); phases.push(...rest); @@ -395,6 +422,7 @@ function seed(route: AnyRoute): AnyRoute { phases: [{ params: {}, routes: [], + values: [], }], }; } @@ -431,7 +459,7 @@ function unexpected( segment.tokens.map((token) => token.index), ); - return Array.from(state.rest) + return Array.from(state.rest.tokens) .filter((token) => members.has(token.index)) .map((token) => ({ message: `unexpected ${JSON.stringify(token.text)}`, diff --git a/lib/rest.ts b/lib/rest.ts new file mode 100644 index 0000000..34c64b8 --- /dev/null +++ b/lib/rest.ts @@ -0,0 +1,8 @@ +import type { Symbol } from "./read.ts"; +import type { Tokenizer } from "./tokenizer.ts"; +import type { Values } from "./values.ts"; + +export interface Rest { + readonly tokens: Tokenizer; + readonly values: Values; +} diff --git a/lib/route.ts b/lib/route.ts index b23a711..9a8e39c 100644 --- a/lib/route.ts +++ b/lib/route.ts @@ -1112,6 +1112,7 @@ export function route( phases: [{ params: {}, routes: [], + values: [], }], }; return extend(elements)(zero); diff --git a/lib/types.ts b/lib/types.ts index 09b8eb2..ba95b11 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -2,6 +2,7 @@ import type { StandardSchemaV1 } from "@standard-schema/spec"; import type { Literal } from "./tokenize.ts"; import type { Param } from "./param.ts"; import type { Result } from "./result.ts"; +import type { ValueSource } from "./values.ts"; export type Issue = StandardSchemaV1.Issue; export type Schema = StandardSchemaV1; @@ -41,6 +42,7 @@ export type Next< > = { readonly params: Params; readonly routes: Routes; + readonly values: readonly ValueSource[]; readonly resolver: ( requirement: T, ) => (input: AnyRoute) => AnyRoute; @@ -52,6 +54,7 @@ export type Done< > = { readonly params: Params; readonly routes: Routes; + readonly values: readonly ValueSource[]; }; export type Params = { @@ -117,6 +120,7 @@ export interface AnyRoute extends Definition { export interface AnyPhase { readonly params: Params; readonly routes: readonly AnyRoute[]; + readonly values: readonly ValueSource[]; readonly resolver?: (requirement: never) => (route: never) => AnyRoute; } @@ -129,6 +133,7 @@ export type Path = readonly string[]; export type Input = { argv: string[]; + values?: readonly ValueSource[]; }; export interface Failure { diff --git a/lib/values.ts b/lib/values.ts index bf6b789..6985722 100644 --- a/lib/values.ts +++ b/lib/values.ts @@ -7,7 +7,7 @@ export type ValueSource = { }; export function withValues( - values: ValueSource[], + values: readonly ValueSource[], ): (route: R) => R { return (route) => { let phases = [...route.phases]; @@ -15,7 +15,7 @@ export function withValues( phases.push({ ...phase, values: phase.values.concat(values), - }) + }); return { ...route, phases, @@ -26,7 +26,7 @@ export function withValues( export type ValueClaim = { result: Maybe<{ source: string; - address: string[]; + address: readonly string[]; value: unknown; }>; rest: Values; @@ -36,12 +36,15 @@ export class Values { mounts: Map; claims: Set; - constructor(mounts: Map = new Map(), claims: Set = new Set()) { + constructor( + mounts: Map = new Map(), + claims: Set = new Set(), + ) { this.mounts = mounts; this.claims = claims; } - mount(path: string[], sources: ValueSource[]): Values { + mount(path: readonly string[], sources: readonly ValueSource[]): Values { if (sources.length === 0) { return this; } @@ -89,21 +92,21 @@ export class Values { } export interface ClaimOptions { - route: string[]; - address: string[]; + route: readonly string[]; + address: readonly string[]; } type ClaimId = string; -function routeId(address: string[]): RoutePath { +function routeId(address: readonly string[]): RoutePath { return `/${address.join("/")}`; } -function claimId(address: string[]): ClaimId { +function claimId(address: readonly string[]): ClaimId { return JSON.stringify(address); } -function find(value: unknown, path: string[]): Maybe { +function find(value: unknown, path: readonly string[]): Maybe { let current: unknown = value; for (let key of path) { diff --git a/mod.ts b/mod.ts index 4005e0e..0f65578 100644 --- a/mod.ts +++ b/mod.ts @@ -13,6 +13,8 @@ export { parse } from "./lib/parse.ts"; export { printErrors, printHelp, printVersion } from "./lib/print.ts"; +export type { Rest } from "./lib/rest.ts"; + export { cli } from "./lib/read.ts"; export type { CLIOptions, @@ -26,6 +28,9 @@ export type { RouteZero } from "./lib/route.ts"; export { toggle } from "./lib/toggle.ts"; +export { withValues } from "./lib/values.ts"; +export type { ValueSource } from "./lib/values.ts"; + export type { Literal } from "./lib/tokenize.ts"; export type { Result } from "./lib/result.ts"; diff --git a/test/bind.test.ts b/test/bind.test.ts index c7c6033..b7fcd9d 100644 --- a/test/bind.test.ts +++ b/test/bind.test.ts @@ -5,15 +5,19 @@ import { bind, bindPhase } from "../lib/bind.ts"; import { name } from "../lib/definition.ts"; import { param, schema } from "../lib/param.ts"; import { cli, type Symbol } from "../lib/read.ts"; +import type { Rest } from "../lib/rest.ts"; import { tokenize } from "../lib/tokenize.ts"; import { Tokenizer } from "../lib/tokenizer.ts"; +import { Values } from "../lib/values.ts"; describe("bind()", () => { describe("absence", () => { it("validates undefined after every source is absent", () => { + let rest = state([]); let { result } = bind({ param: param(name("port"), schema(z.number())), - tokens: symbols([]), + view: rest.tokens, + rest, }); expect(result).toMatchObject({ @@ -23,9 +27,11 @@ describe("bind()", () => { }); it("lets an optional schema accept an absent value", () => { + let rest = state([]); let { result } = bind({ param: param(name("port"), schema(z.number().optional())), - tokens: symbols([]), + view: rest.tokens, + rest, }); expect(result).toEqual({ @@ -36,9 +42,11 @@ describe("bind()", () => { }); it("lets a defaulting schema produce a value from absence", () => { + let rest = state([]); let { result } = bind({ param: param(name("port"), schema(z.number().default(9000))), - tokens: symbols([]), + view: rest.tokens, + rest, }); expect(result).toEqual({ @@ -51,13 +59,15 @@ describe("bind()", () => { describe("candidates", () => { it("tries a later decoding after an earlier one fails validation", () => { + let rest = state(["--digits", "0012"]); let { result } = bind({ param: param( name("digits"), cli(["--digits"]), schema(z.string()), ), - tokens: symbols(["--digits", "0012"]), + view: rest.tokens, + rest, }); expect(result).toEqual({ @@ -70,26 +80,51 @@ describe("bind()", () => { describe("failed reads", () => { it("continues from the remainder of a claimed invalid read", () => { + let input = state(["--port", "--verbose"]); let { result, rest } = bind({ param: param( name("port"), cli(["--port"]), schema(z.number()), ), - tokens: symbols(["--port", "--verbose"]), + view: input.tokens, + rest: input, }); expect(result).toMatchObject({ ok: false, issues: [{ message: "--port requires a value" }], }); - expect(texts(rest)).toEqual(["--verbose"]); + expect(texts(rest.tokens)).toEqual(["--verbose"]); }); }); + it("preserves value sources when claiming CLI tokens", () => { + let values = new Values().mount([], [{ + name: "settings", + value: { port: 9000 }, + }]); + let rest: Rest = { + tokens: symbols(["--port", "9001"]), + values, + }; + let binding = bind({ + param: param( + name("port"), + cli(["--port"]), + schema(z.number()), + ), + view: rest.tokens, + rest, + }); + + expect(binding.rest.values).toBe(values); + expect(texts(binding.rest.tokens)).toEqual([]); + }); + describe("phase", () => { it("advances a cursor consumed by the phase", () => { - let tokens = symbols([ + let rest = state([ "--foo", "--bar", "-c", @@ -98,7 +133,7 @@ describe("bind()", () => { "--x", "--y=z", ]); - let values = Array.from(tokens); + let items = Array.from(rest.tokens); let config = param( name("config"), cli(["-c"]), @@ -113,12 +148,13 @@ describe("bind()", () => { phase: { params: { config, x }, routes: [], + values: [], }, segment: { - tokens: values, - cursor: values.find((token) => token.text === "app.json")?.index, + range: { start: -1 }, + cursor: items.find((token) => token.text === "app.json")?.index, }, - tokens, + rest, }); expect(binding).toMatchObject({ @@ -127,9 +163,9 @@ describe("bind()", () => { config: "app.json", x: undefined, }, - cursor: values.find((token) => token.text === "auth0")?.index, + cursor: items.find((token) => token.text === "auth0")?.index, }); - expect(texts(binding.rest)).toEqual([ + expect(texts(binding.rest.tokens)).toEqual([ "--foo", "--bar", "auth0", @@ -139,14 +175,14 @@ describe("bind()", () => { }); it("retries absent parameters after another parameter advances the cursor", () => { - let tokens = symbols([ + let rest = state([ "--target", "local", "--port", "9000", "auth0", ]); - let values = Array.from(tokens); + let items = Array.from(rest.tokens); let port = param( name("port"), cli(["--port"]), @@ -162,12 +198,13 @@ describe("bind()", () => { // Port deliberately comes first. It is initially beyond the cursor. params: { port, target }, routes: [], + values: [], }, segment: { - tokens: values, - cursor: values.find((token) => token.text === "local")?.index, + range: { start: -1 }, + cursor: items.find((token) => token.text === "local")?.index, }, - tokens, + rest, }); expect(binding).toMatchObject({ @@ -176,9 +213,9 @@ describe("bind()", () => { port: 9000, target: "local", }, - cursor: values.find((token) => token.text === "auth0")?.index, + cursor: items.find((token) => token.text === "auth0")?.index, }); - expect(texts(binding.rest)).toEqual(["auth0"]); + expect(texts(binding.rest.tokens)).toEqual(["auth0"]); }); }); }); @@ -192,6 +229,13 @@ function symbols(argv: string[]): Tokenizer { return new Tokenizer(tokens); } +function state(argv: string[]): Rest { + return { + tokens: symbols(argv), + values: new Values(), + }; +} + function texts(tokens: Iterable): string[] { return Array.from(tokens, (token) => token.text); } From f0b379f8970f4ea1beafe885dd3a87442d160cb0 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Thu, 3 Sep 2026 16:07:34 -0500 Subject: [PATCH 5/6] =?UTF-8?q?=E2=9C=A8=20Implement=20source-aware=20phas?= =?UTF-8?q?e=20binding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/binding.md | 13 +- docs/insights.md | 1 + lib/bind.ts | 286 ++++++++++++++++++++++++---------------- lib/checkpoint.ts | 14 ++ lib/parse.ts | 14 +- lib/tokenizer.ts | 4 +- mod.ts | 2 + test/bind.test.ts | 188 ++++++++++++++++++--------- test/dynamic.test.ts | 23 ---- test/parse.test.ts | 30 +++++ test/tokenizer.test.ts | 14 ++ test/value.test.ts | 289 ++++++++++++++++++++++++++++++++++++++++- 12 files changed, 660 insertions(+), 218 deletions(-) create mode 100644 lib/checkpoint.ts diff --git a/docs/binding.md b/docs/binding.md index 0c5d772..1325f4f 100644 --- a/docs/binding.md +++ b/docs/binding.md @@ -38,10 +38,10 @@ interface Rest { A source attempt must distinguish absence from a captured value that failed: ```ts -type Attempt = Maybe<{ +interface Binding { readonly rest: Rest; readonly result: Result; -}>; +} ``` The outcomes mean: @@ -70,9 +70,9 @@ Source adapters share the attempt shape while retaining their own capture mechanics: ```ts -fromCLI({ param, view, rest }): Attempt; -fromValues({ param, route, rest }): Attempt; -fromEnv({ param, route, rest }): Attempt; +fromCLI({ param, view, rest }): Maybe>; +fromValues({ param, route, rest }): Maybe>; +fromEnv({ param, route, rest }): Maybe>; ``` CLI and Env decode captured text before validating it. Values are already @@ -206,6 +206,7 @@ ever-deepening stack of tokenizer iterators. The remaining invariants are: - Stable global token indices identify every claim. +- Claims leave holes; they never make originally non-adjacent tokens a pair. - `through` is inclusive. - An absent `through` means the full route segment is visible. - Finding the first remaining word is CLI-binding policy, not generic tokenizer @@ -240,7 +241,7 @@ function bindPhase(options: BindPhaseOptions): PhaseBinding { function accept( param: Param, - attempt: Attempt, + attempt: Maybe>, ): void { if (!attempt.exists) { return; diff --git a/docs/insights.md b/docs/insights.md index 35c9862..55059ad 100644 --- a/docs/insights.md +++ b/docs/insights.md @@ -50,6 +50,7 @@ The detailed phase-binding design is recorded in [Binding](./binding.md). - Tokenization describes syntax only; bindings assign semantic meaning. - Input is immutable. - Tokens retain stable global indices and original order. +- Claims never make originally non-adjacent tokens adjacent. - Parents establish scope; children claim only within that scope. - Available input flows down; claims, issues, and models flow up. - Claims plus remainder conserve the original input: nothing disappears or is diff --git a/lib/bind.ts b/lib/bind.ts index ea621de..2a7a9e6 100644 --- a/lib/bind.ts +++ b/lib/bind.ts @@ -1,145 +1,180 @@ +import type { Maybe } from "./maybe.ts"; import type { Param } from "./param.ts"; import type { Symbol } from "./read.ts"; import type { Rest } from "./rest.ts"; import type { Result } from "./result.ts"; -import type { Flag, Setter, Word } from "./tokenize.ts"; +import type { Word } from "./tokenize.ts"; import type { TokenInput, Tokenizer, TokenRange } from "./tokenizer.ts"; -import type { AnyPhase, Issue } from "./types.ts"; +import type { AnyPhase, Issue, Path } from "./types.ts"; export interface Binding { - rest: Rest; - result: Result; + readonly rest: Rest; + readonly result: Result; } export interface PhaseBinding { - rest: Rest; - model: Record; - issues: Issue[]; - valid: boolean; - cursor?: number; + readonly rest: Rest; + readonly model: Record; + readonly issues: Issue[]; + readonly valid: boolean; } export interface PhaseSegment { readonly range: TokenRange; - readonly cursor?: number; + readonly path: Path; } -export function bind>(options: { - param: P; - view: TokenInput; - rest: Rest; -}): Binding { +export function fromCLI(options: { + readonly param: Param; + readonly view: TokenInput; + readonly rest: Rest; +}): Maybe> { let { param, view, rest } = options; let path = [param.name]; - let cli = param.cli(view); - let next = { - ...rest, - tokens: cli.claim.rest, - }; + let read = param.cli(view); - if (cli.result.ok) { - let read = cli.result.value; - let value: unknown = read.exists ? read.value : undefined; - - let candidates = typeof value === "string" ? param.decode(value) : [value]; - - if (candidates.length === 0) { - return { - rest: next, - result: { - ok: false, - issues: [{ - message: `unable to decode ${JSON.stringify(value)}`, - path, - }], + if (!read.result.ok) { + return { + exists: true, + value: { + rest: { + ...rest, + tokens: read.claim.rest, }, - }; - } + result: read.result, + }, + }; + } - let issues: readonly Issue[] | undefined = undefined; - for (let candidate of candidates) { - let result = validate(param, candidate, path); - if (result.ok) { - return { - rest: next, - result, - }; - } else { - issues = issues ?? result.issues; - } - } + if (!read.result.value.exists) { + return { exists: false }; + } - return { - rest: next, - result: { - ok: false, - issues: issues ?? [], + let value = read.result.value.value; + let candidates = typeof value === "string" ? param.decode(value) : [value]; + let result = merge( + decode(param, value, candidates, path), + read.result.issues, + ); + + return { + exists: true, + value: { + rest: { + ...rest, + tokens: read.claim.rest, }, - }; + result, + }, + }; +} + +export function fromValues(options: { + readonly param: Param; + readonly route: Path; + readonly rest: Rest; +}): Maybe> { + let { param, route, rest } = options; + let claim = rest.values.claim({ + route, + address: [param.name], + }); + + if (!claim.result.exists) { + return { exists: false }; } return { - rest: next, - result: { - ok: false, - issues: cli.result.issues, + exists: true, + value: { + rest: { + ...rest, + values: claim.rest, + }, + result: validate(param, claim.result.value.value, [param.name]), }, }; } export function bindPhase(options: { - phase: AnyPhase; - segment: PhaseSegment; - rest: Rest; + readonly phase: AnyPhase; + readonly segment: PhaseSegment; + readonly rest: Rest; }): PhaseBinding { let { phase, segment } = options; let rest = options.rest; let params = Object.values(phase.params) as Param[]; - let cursor = segment.cursor; + let pending = new Map(params.map((param) => [param.name, param])); let results = new Map>(); - let settled = new Set(); + function accept( + param: Param, + attempt: Maybe>, + ): void { + if (!attempt.exists) { + return; + } + + rest = attempt.value.rest; + results.set(param.name, attempt.value.result); + pending.delete(param.name); + } + + // CLI visibility grows whenever a parameter consumes the current horizon. + // Keep retrying provisional misses until a complete sweep leaves the first + // remaining word unchanged. while (true) { - for (let param of params) { - if (settled.has(param.name)) { - continue; - } + let before = first(rest.tokens, segment.range); - let tokens = rest.tokens; - let binding = bind({ + for (let param of pending.values()) { + let attempt = fromCLI({ param, - view: tokens.view({ + view: rest.tokens.view({ range: segment.range, - through: cursor, + through: before?.index, }), rest, }); - results.set(param.name, binding.result); - rest = binding.rest; - - if (rest.tokens !== tokens) { - settled.add(param.name); - } + accept(param, attempt); } - if (cursor === undefined || has(rest.tokens, segment.range, cursor)) { + let after = first(rest.tokens, segment.range); + if (after?.index === before?.index) { break; } + } + + // Address sources have stable visibility once the route is known. They are + // tried only after CLI has reached its fixed point so a provisional CLI miss + // cannot let a lower-priority source settle the parameter too early. + for (let source of [fromValues]) { + for (let param of pending.values()) { + accept( + param, + source({ + param, + route: segment.path, + rest, + }), + ); + } + } - cursor = next(rest.tokens, segment.range, cursor); + // Only total absence reaches the schema as undefined. This is where required, + // optional, and defaulting parameters diverge. + for (let param of pending.values()) { + results.set(param.name, validate(param, undefined, [param.name])); } let model: Record = {}; let issues: Issue[] = []; let valid = true; + // Collect in declaration order, independent of the source or sweep that + // settled each parameter. for (let param of params) { - let result = results.get(param.name); - if (!result) { - continue; - } - + let result = results.get(param.name)!; issues.push(...result.issues ?? []); if (result.ok) { @@ -149,11 +184,43 @@ export function bindPhase(options: { } } - return { rest, model, issues, valid, cursor }; + return { rest, model, issues, valid }; } -function validate>( - param: P, +function decode( + param: Param, + value: unknown, + candidates: unknown[], + path: string[], +): Result { + if (candidates.length === 0) { + return { + ok: false, + issues: [{ + message: `unable to decode ${JSON.stringify(value)}`, + path, + }], + }; + } + + let issues: readonly Issue[] | undefined; + + for (let candidate of candidates) { + let result = validate(param, candidate, path); + if (result.ok) { + return result; + } + issues = issues ?? result.issues; + } + + return { + ok: false, + issues: issues ?? [], + }; +} + +function validate( + param: Param, value: unknown, path: string[], ): Result { @@ -177,36 +244,33 @@ function validate>( path, })), }; - } else { - return { - ok: true, - issues: [], - value: validated.value, - }; } + + return { + ok: true, + issues: [], + value: validated.value, + }; } -function has( - tokens: Tokenizer, - range: TokenRange, - index: number, -): boolean { - for (let token of tokens.view({ range })) { - if (token.index === index) { - return true; - } +function merge( + result: Result, + issues: readonly Issue[] | undefined, +): Result { + if (!issues || issues.length === 0) { + return result; } - return false; + + return { + ...result, + issues: [...issues, ...(result.issues ?? [])], + }; } -function next( - tokens: Tokenizer, - range: TokenRange, - index: number, -): number | undefined { +function first(tokens: Tokenizer, range: TokenRange): Word | undefined { for (let token of tokens.view({ range })) { - if (token.index > index && token.type === "word") { - return token.index; + if (token.type === "word") { + return token; } } } diff --git a/lib/checkpoint.ts b/lib/checkpoint.ts new file mode 100644 index 0000000..49e3b38 --- /dev/null +++ b/lib/checkpoint.ts @@ -0,0 +1,14 @@ +import { dynamic, type Seed } from "./dynamic.ts"; +import type { AnyRoute } from "./types.ts"; +import { type ValueSource, withValues } from "./values.ts"; + +export function checkpoint(): ( + route: Before, +) => ReturnType< + ReturnType>> +> { + return (route: Before) => + dynamic>( + (values) => withValues(values), + )(route); +} diff --git a/lib/parse.ts b/lib/parse.ts index 0da56f5..299ef65 100644 --- a/lib/parse.ts +++ b/lib/parse.ts @@ -112,8 +112,8 @@ function resume( // Bind only this phase, within this segment's token view. // - // For an open dynamic segment, a binding may consume the cursor but may - // not leap over it. For a committed segment, `end` is the hard boundary. + // For an open dynamic segment, binding may consume the horizon but may not + // look beyond it. For a committed segment, `end` is the hard boundary. let binding = bindPhase({ phase, segment: { @@ -121,7 +121,7 @@ function resume( start: segment.start, end: segment.end, }, - cursor: segment.cursor, + path: segment.path, }, rest: state.rest, }); @@ -136,7 +136,6 @@ function resume( ...segment.issues, ...binding.issues, ], - cursor: binding.cursor, }; state = { @@ -268,7 +267,6 @@ interface Segment { tokens: Symbol[]; start: number; end?: number; - cursor?: number; model: Record; issues: Issue[]; history: AnyPhase[]; @@ -289,11 +287,6 @@ function search(state: ParserState): ParserState { }); if (!selector) { - let cursor = segment.tokens.find((token) => { - return remaining.has(token.index) && token.type === "word"; - })?.index; - - segments[index] = { ...segment, cursor }; return { ...state, segments, rest }; } @@ -306,7 +299,6 @@ function search(state: ParserState): ParserState { ...segment, tokens: before, end: selector.index, - cursor: undefined, }; segments.push({ id: `/${path.join("/")}`, diff --git a/lib/tokenizer.ts b/lib/tokenizer.ts index 9469e0e..12977a7 100644 --- a/lib/tokenizer.ts +++ b/lib/tokenizer.ts @@ -59,7 +59,7 @@ export class Tokenizer implements TokenInput { previous = token; continue; } - if (match(previous, token)) { + if (token.index === previous.index + 1 && match(previous, token)) { return { tokens: [previous, token], rest: remainder(this, [previous.index, token.index]), @@ -136,7 +136,7 @@ class View implements TokenInput { previous = token; continue; } - if (match(previous, token)) { + if (token.index === previous.index + 1 && match(previous, token)) { return { tokens: [previous, token], rest: remainder(this.source, [previous.index, token.index]), diff --git a/mod.ts b/mod.ts index 0f65578..99507c0 100644 --- a/mod.ts +++ b/mod.ts @@ -2,6 +2,8 @@ export { extend } from "./lib/extend.ts"; export { command } from "./lib/command.ts"; export type { CommandZero } from "./lib/command.ts"; +export { checkpoint } from "./lib/checkpoint.ts"; + export { description, name } from "./lib/definition.ts"; export { option } from "./lib/option.ts"; diff --git a/test/bind.test.ts b/test/bind.test.ts index b7fcd9d..5925465 100644 --- a/test/bind.test.ts +++ b/test/bind.test.ts @@ -1,66 +1,78 @@ import { expect } from "@std/expect"; import { describe, it } from "@std/testing/bdd"; import * as z from "zod"; -import { bind, bindPhase } from "../lib/bind.ts"; +import { type Binding, bindPhase, fromCLI } from "../lib/bind.ts"; import { name } from "../lib/definition.ts"; -import { param, schema } from "../lib/param.ts"; +import type { Maybe } from "../lib/maybe.ts"; +import { type Param, param, schema } from "../lib/param.ts"; import { cli, type Symbol } from "../lib/read.ts"; import type { Rest } from "../lib/rest.ts"; import { tokenize } from "../lib/tokenize.ts"; import { Tokenizer } from "../lib/tokenizer.ts"; import { Values } from "../lib/values.ts"; -describe("bind()", () => { +describe("binding", () => { describe("absence", () => { it("validates undefined after every source is absent", () => { - let rest = state([]); - let { result } = bind({ - param: param(name("port"), schema(z.number())), - view: rest.tokens, - rest, + let port = param(name("port"), schema(z.number())); + let result = bindPhase({ + phase: phase({ port }), + segment: segment(), + rest: state([]), }); expect(result).toMatchObject({ - ok: false, + valid: false, issues: [{ path: ["port"] }], }); }); it("lets an optional schema accept an absent value", () => { - let rest = state([]); - let { result } = bind({ - param: param(name("port"), schema(z.number().optional())), - view: rest.tokens, - rest, + let port = param(name("port"), schema(z.number().optional())); + let result = bindPhase({ + phase: phase({ port }), + segment: segment(), + rest: state([]), }); - expect(result).toEqual({ - ok: true, - value: undefined, + expect(result).toMatchObject({ + valid: true, + model: { port: undefined }, issues: [], }); }); it("lets a defaulting schema produce a value from absence", () => { + let port = param(name("port"), schema(z.number().default(9000))); + let result = bindPhase({ + phase: phase({ port }), + segment: segment(), + rest: state([]), + }); + + expect(result).toMatchObject({ + valid: true, + model: { port: 9000 }, + issues: [], + }); + }); + + it("represents a source miss without validating undefined", () => { let rest = state([]); - let { result } = bind({ - param: param(name("port"), schema(z.number().default(9000))), + let attempt = fromCLI({ + param: param(name("port"), schema(z.number())), view: rest.tokens, rest, }); - expect(result).toEqual({ - ok: true, - value: 9000, - issues: [], - }); + expect(attempt).toEqual({ exists: false }); }); }); describe("candidates", () => { it("tries a later decoding after an earlier one fails validation", () => { let rest = state(["--digits", "0012"]); - let { result } = bind({ + let attempt = fromCLI({ param: param( name("digits"), cli(["--digits"]), @@ -70,10 +82,16 @@ describe("bind()", () => { rest, }); - expect(result).toEqual({ - ok: true, - value: "0012", - issues: [], + expectType>>>(true); + expect(attempt).toMatchObject({ + exists: true, + value: { + result: { + ok: true, + value: "0012", + issues: [], + }, + }, }); }); }); @@ -81,7 +99,7 @@ describe("bind()", () => { describe("failed reads", () => { it("continues from the remainder of a claimed invalid read", () => { let input = state(["--port", "--verbose"]); - let { result, rest } = bind({ + let attempt = fromCLI({ param: param( name("port"), cli(["--port"]), @@ -91,11 +109,47 @@ describe("bind()", () => { rest: input, }); - expect(result).toMatchObject({ + assertAttempt(attempt); + expect(attempt.value.result).toMatchObject({ ok: false, issues: [{ message: "--port requires a value" }], }); - expect(texts(rest.tokens)).toEqual(["--verbose"]); + expect(texts(attempt.value.rest.tokens)).toEqual(["--verbose"]); + }); + }); + + it("preserves issues from a successful read", () => { + let base = param( + name("port"), + cli(["--port"]), + schema(z.number()), + ); + let read = base.cli; + let port = { + ...base, + cli(tokens: Parameters[0]) { + let capture = read(tokens); + return capture.result.ok + ? { + ...capture, + result: { + ...capture.result, + issues: [{ message: "--port is deprecated" }], + }, + } + : capture; + }, + }; + let result = bindPhase({ + phase: phase({ port }), + segment: segment(), + rest: state(["--port", "9001"]), + }); + + expect(result).toMatchObject({ + valid: true, + model: { port: 9001 }, + issues: [{ message: "--port is deprecated" }], }); }); @@ -108,7 +162,7 @@ describe("bind()", () => { tokens: symbols(["--port", "9001"]), values, }; - let binding = bind({ + let attempt = fromCLI({ param: param( name("port"), cli(["--port"]), @@ -118,12 +172,13 @@ describe("bind()", () => { rest, }); - expect(binding.rest.values).toBe(values); - expect(texts(binding.rest.tokens)).toEqual([]); + assertAttempt(attempt); + expect(attempt.value.rest.values).toBe(values); + expect(texts(attempt.value.rest.tokens)).toEqual([]); }); describe("phase", () => { - it("advances a cursor consumed by the phase", () => { + it("advances the horizon when the phase consumes it", () => { let rest = state([ "--foo", "--bar", @@ -133,7 +188,6 @@ describe("bind()", () => { "--x", "--y=z", ]); - let items = Array.from(rest.tokens); let config = param( name("config"), cli(["-c"]), @@ -145,15 +199,8 @@ describe("bind()", () => { schema(z.string().optional()), ); let binding = bindPhase({ - phase: { - params: { config, x }, - routes: [], - values: [], - }, - segment: { - range: { start: -1 }, - cursor: items.find((token) => token.text === "app.json")?.index, - }, + phase: phase({ config, x }), + segment: segment(), rest, }); @@ -163,7 +210,6 @@ describe("bind()", () => { config: "app.json", x: undefined, }, - cursor: items.find((token) => token.text === "auth0")?.index, }); expect(texts(binding.rest.tokens)).toEqual([ "--foo", @@ -174,7 +220,7 @@ describe("bind()", () => { ]); }); - it("retries absent parameters after another parameter advances the cursor", () => { + it("retries absent parameters after another parameter advances the horizon", () => { let rest = state([ "--target", "local", @@ -182,7 +228,6 @@ describe("bind()", () => { "9000", "auth0", ]); - let items = Array.from(rest.tokens); let port = param( name("port"), cli(["--port"]), @@ -194,16 +239,9 @@ describe("bind()", () => { schema(z.string()), ); let binding = bindPhase({ - phase: { - // Port deliberately comes first. It is initially beyond the cursor. - params: { port, target }, - routes: [], - values: [], - }, - segment: { - range: { start: -1 }, - cursor: items.find((token) => token.text === "local")?.index, - }, + // Port deliberately comes first. It is initially beyond the horizon. + phase: phase({ port, target }), + segment: segment(), rest, }); @@ -213,7 +251,6 @@ describe("bind()", () => { port: 9000, target: "local", }, - cursor: items.find((token) => token.text === "auth0")?.index, }); expect(texts(binding.rest.tokens)).toEqual(["auth0"]); }); @@ -236,6 +273,37 @@ function state(argv: string[]): Rest { }; } +function phase(params: Record>) { + return { + params, + routes: [], + values: [], + }; +} + +function segment() { + return { + range: { start: -1 }, + path: [], + }; +} + +function assertAttempt( + attempt: Maybe>, +): asserts attempt is { readonly exists: true; readonly value: Binding } { + expect(attempt).toMatchObject({ exists: true }); +} + function texts(tokens: Iterable): string[] { return Array.from(tokens, (token) => token.text); } + +type Equal = (() => T extends L ? 1 : 2) extends + (() => T extends R ? 1 : 2) + ? (() => T extends R ? 1 : 2) extends (() => T extends L ? 1 : 2) ? true + : false + : false; + +function expectType(_value: T): void { + // Compile-time assertion. +} diff --git a/test/dynamic.test.ts b/test/dynamic.test.ts index 4d1d3a2..4a82c8f 100644 --- a/test/dynamic.test.ts +++ b/test/dynamic.test.ts @@ -751,29 +751,6 @@ Options: }); describe("future sources", () => { - // Input does not expose value or environment sources yet. - it.skip("lets a later phase claim an existing value source", () => { - // let app = command( - // name("simulacrum"), - // dynamic((_config: Config) => - // extend(option(name("b"), prop("b"))) - // ), - // ); - // let first = parse(app, { - // argv: [], - // values: [{ name: "settings", value: { b: "two" } }], - // }); - // increment(first, {}); - // let result = first.resume({ - // ok: true, - // value: { services: [] }, - // }); - // expect(result).toMatchObject({ - // method: "execute", - // model: { b: "two" }, - // }); - }); - it.skip("lets a later phase claim an existing environment source", () => { // let app = command( // name("simulacrum"), diff --git a/test/parse.test.ts b/test/parse.test.ts index d980eb9..21a0338 100644 --- a/test/parse.test.ts +++ b/test/parse.test.ts @@ -366,6 +366,36 @@ describe("parse()", () => { ).toHaveModels({ "/": { dryRun: false } }); }); + it("does not let declaration order change option adjacency", () => { + let portFirst = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + toggle(name("verbose")), + ); + let verboseFirst = command( + name("simulacrum"), + toggle(name("verbose")), + option(name("port"), schema(type("number"))), + ); + let expected = { + ok: false, + code: "unprocessable-content", + route: "/", + issues: [ + { message: 'unexpected "9000"' }, + { message: "--port requires a value" }, + ], + }; + + expect(parse(portFirst, { + argv: ["--port", "--verbose", "9000"], + })).toMatchObject(expected); + + expect(parse(verboseFirst, { + argv: ["--port", "--verbose", "9000"], + })).toMatchObject(expected); + }); + it("reports several surplus arguments as a binding error", () => { let result = exec("simulacrum databaes clean"); diff --git a/test/tokenizer.test.ts b/test/tokenizer.test.ts index ac2a457..f43cb21 100644 --- a/test/tokenizer.test.ts +++ b/test/tokenizer.test.ts @@ -79,6 +79,20 @@ describe("Tokenizer", () => { expect(tested).toEqual([1, 2]); }); + it("does not pair tokens across an existing claim", () => { + let tokenizer = new Tokenizer(tokenize([ + "--port", + "--verbose", + "9000", + ])); + let rest = tokenizer.claimOne((token) => token.index === 1).rest; + let pair = rest.claimPair((name, value) => { + return name.text === "--port" && value.text === "9000"; + }); + + expect(pair.tokens).toEqual([]); + }); + describe("view()", () => { it("shows only tokens in its range through its inclusive horizon", () => { let tokenizer = new Tokenizer(tokenize([ diff --git a/test/value.test.ts b/test/value.test.ts index a61e156..f289340 100644 --- a/test/value.test.ts +++ b/test/value.test.ts @@ -10,9 +10,10 @@ import { schema } from "../lib/param.ts"; import { parse } from "../lib/parse.ts"; import { routes } from "../lib/route.ts"; import type { Parse } from "../lib/types.ts"; +import { withValues } from "../lib/values.ts"; describe("value sources", () => { - it.skip("binds top-level properties to root parameters", () => { + it("binds top-level properties to root parameters", () => { let app = command( name("simulacrum"), option(name("port"), schema(type("number"))), @@ -48,7 +49,7 @@ describe("value sources", () => { }); }); - it.skip("scopes nested properties by route name", () => { + it("scopes nested properties by route name", () => { let auth0 = command( name("auth0"), option(name("delay"), schema(type("number"))), @@ -92,7 +93,7 @@ describe("value sources", () => { }); }); - it.skip("combines disjoint properties from several value sources", () => { + it("combines disjoint properties from several value sources", () => { let app = command( name("simulacrum"), option(name("host"), schema(type("string"))), @@ -124,7 +125,7 @@ describe("value sources", () => { }); }); - it.skip("validates values without applying the CLI decoder", () => { + it("validates values without applying the CLI decoder", () => { let app = command( name("simulacrum"), option(name("port"), schema(type("number"))), @@ -146,7 +147,243 @@ describe("value sources", () => { }); }); - it.skip("keeps values available to parameters introduced by a later phase", () => { + it("lets CLI override a value source", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + ); + let result = parse(app, { + argv: ["--port", "9001"], + values: [{ + name: "settings", + value: { port: "not a number" }, + }], + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { port: 9001 }, + }); + }); + + it("retries CLI before falling back to values", () => { + let app = command( + name("simulacrum"), + // Port cannot initially see beyond the value consumed by target. + option(name("port"), schema(type("number"))), + option(name("target"), schema(type("string"))), + ); + let result = parse(app, { + argv: ["--target", "local", "--port", "9001"], + values: [{ + name: "settings", + value: { port: 8000 }, + }], + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { + port: 9001, + target: "local", + }, + }); + }); + + it("does not hide an incomplete CLI option with a value", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + ); + let result = parse(app, { + argv: ["--port"], + values: [{ + name: "settings", + value: { port: 9001 }, + }], + }); + + expect(result).toMatchObject({ + ok: false, + code: "unprocessable-content", + route: "/", + issues: [{ message: "--port requires a value" }], + }); + }); + + it("does not hide an invalid CLI value with a value", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + ); + let result = parse(app, { + argv: ["--port", "nope"], + values: [{ + name: "settings", + value: { port: 9001 }, + }], + }); + + expect(result).toMatchObject({ + ok: false, + code: "unprocessable-content", + route: "/", + issues: [{ path: ["port"] }], + }); + }); + + it("does not hide an invalid value with a later value", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + ); + let result = parse(app, { + argv: [], + values: [ + { + name: "settings", + value: { port: "nope" }, + }, + { + name: "defaults", + value: { port: 9001 }, + }, + ], + }); + + expect(result).toMatchObject({ + ok: false, + code: "unprocessable-content", + route: "/", + issues: [{ path: ["port"] }], + }); + }); + + it("treats an explicit undefined value as present", () => { + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number | undefined"))), + ); + let result = parse(app, { + argv: [], + values: [ + { + name: "settings", + value: { port: undefined }, + }, + { + name: "defaults", + value: { port: 9001 }, + }, + ], + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { port: undefined }, + }); + }); + + it("uses values declared on a phase", () => { + let app = command( + name("simulacrum"), + withValues([{ + name: "defaults", + value: { port: 9001 }, + }]), + option(name("port"), schema(type("number"))), + ); + let result = parse(app, { argv: [] }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { port: 9001 }, + }); + }); + + it("prefers values mounted on the selected route", () => { + let auth0 = command( + name("auth0"), + withValues([{ + name: "auth0.json", + value: { port: 9003 }, + }]), + option(name("port"), schema(type("number"))), + ); + let app = command( + name("simulacrum"), + withValues([{ + name: "settings", + value: { auth0: { port: 9001 } }, + }]), + routes(auth0), + ); + let result = parse(app, { argv: ["auth0"] }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/auth0", + model: { port: 9003 }, + }); + }); + + it("keeps parent values separate from a dynamically discovered child", () => { + let auth0 = command( + name("auth0"), + option(name("port"), schema(type("number"))), + ); + let app = command( + name("simulacrum"), + option(name("port"), schema(type("number"))), + option(name("target"), schema(type("string"))), + dynamic((_plugins: Plugins) => extend(routes(auth0))), + ); + let increment = parse(app, { + argv: ["--target", "local", "auth0", "--port", "9001"], + values: [{ + name: "settings", + value: { port: 8000 }, + }], + }); + + assertIncrement(increment); + expect(increment).toMatchObject({ + model: { + port: 8000, + target: "local", + }, + }); + + let result = increment.resume({ + ok: true, + value: { names: ["auth0"] }, + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/auth0", + model: { port: 9001 }, + models: { + "/": { + port: 8000, + target: "local", + }, + "/auth0": { port: 9001 }, + }, + }); + }); + + it("keeps values available to parameters introduced by a later phase", () => { let app = command( name("simulacrum"), dynamic((_plugins: Plugins) => @@ -181,6 +418,48 @@ describe("value sources", () => { }); }); + it("keeps dynamically introduced values available downstream", () => { + let app = command( + name("simulacrum"), + dynamic((_plugins: Plugins) => + extend( + withValues([{ + name: "settings", + value: { domain: "auth0.local" }, + }]), + dynamic((_services: Plugins) => + extend( + option(name("domain"), schema(type("string"))), + ) + ), + ) + ), + ); + let first = parse(app, { argv: [] }); + + assertIncrement(first); + let second = first.resume({ + ok: true, + value: { names: ["config"] }, + }); + + assertIncrement(second); + let result = second.resume({ + ok: true, + value: { names: ["auth0"] }, + }); + + expect(result).toMatchObject({ + ok: true, + method: "execute", + route: "/", + model: { domain: "auth0.local" }, + models: { + "/": { domain: "auth0.local" }, + }, + }); + }); + it.skip("handles a value key shared by a parameter and child route", () => { let auth0 = command( name("auth0"), From 3580c786e6e45a2304eb1114a375fc77309f69e0 Mon Sep 17 00:00:00 2001 From: Charles Lowell Date: Tue, 8 Sep 2026 20:33:46 -0500 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=8E=A8=20Format=20binding=20architect?= =?UTF-8?q?ure=20documentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/binding.md | 76 +++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/docs/binding.md b/docs/binding.md index 1325f4f..61735fd 100644 --- a/docs/binding.md +++ b/docs/binding.md @@ -3,14 +3,14 @@ `bindPhase()` is a source scheduler, not a source interpreter. CLI, Values, and Env all attempt to supply values for parameters, but they do -not traverse their inputs in the same way. CLI is positional: its safely -visible input can grow as options and arguments consume words. Values and Env -are address sources: once the route and parameter address are known, their +not traverse their inputs in the same way. CLI is positional: its safely visible +input can grow as options and arguments consume words. Values and Env are +address sources: once the route and parameter address are known, their visibility is stable. -The binding algorithm should preserve that distinction while giving every -source the same attempt protocol. The intended precedence is visible directly -in its structure: +The binding algorithm should preserve that distinction while giving every source +the same attempt protocol. The intended precedence is visible directly in its +structure: ```text CLI fixed point @@ -55,8 +55,8 @@ The outcomes mean: are blocked. A malformed CLI option is therefore an existing, failed attempt because it -claimed input. A decoding or schema error from a higher-priority source must -not disappear by silently selecting a lower-priority value. +claimed input. A decoding or schema error from a higher-priority source must not +disappear by silently selecting a lower-priority value. Explicit `undefined` from a JavaScript value source is also present. It must be distinguished from an absent property, must shadow lower-priority sources, and @@ -88,10 +88,10 @@ The **horizon** is the first remaining word in the current route segment. It is the inclusive boundary of safe lookahead. This is not a cursor: it does not identify the current consumption position. -The horizon itself must be visible so an option or positional argument can -prove that the word is data by claiming it. The entire suffix after it remains -hidden because, if the horizon becomes a child selector, every following -option belongs to that child. +The horizon itself must be visible so an option or positional argument can prove +that the word is data by claiming it. The entire suffix after it remains hidden +because, if the horizon becomes a child selector, every following option belongs +to that child. ### Dynamic route discovery @@ -118,8 +118,8 @@ view: [---] |--- hidden ---| ``` Nothing in the current phase claims `auth0`, so the horizon remains unchanged -after a complete parameter sweep. CLI binding has reached a fixed point and -must stop. +after a complete parameter sweep. CLI binding has reached a fixed point and must +stop. If `resume()` introduces an `auth0` route, route search can now claim index 2 as its selector. The child receives `--port 9001`. Without the horizon, a parent @@ -157,8 +157,8 @@ moves to `auth0`: horizon ``` -No parameter claims it, so the horizon is stable and CLI binding stops. This -is a fixed-point calculation rather than an ordinary left-to-right cursor. +No parameter claims it, so the horizon is stable and CLI binding stops. This is +a fixed-point calculation rather than an ordinary left-to-right cursor. Route search always runs before phase binding. A currently discoverable route therefore beats an option value. A route introduced only after the current @@ -168,9 +168,9 @@ the unambiguous escape hatch. ## Tokenizer support -There is one global tokenizer and one global claim ledger for a parse. A -binding phase needs borrowed, bounded views into that tokenizer rather than -temporary tokenizers whose claims later have to be reconstructed. +There is one global tokenizer and one global claim ledger for a parse. A binding +phase needs borrowed, bounded views into that tokenizer rather than temporary +tokenizers whose claims later have to be reconstructed. The central addition is a first-class view: @@ -189,19 +189,18 @@ class Tokenizer implements TokenInput { } ``` -A view changes visibility only. It never owns claims. Every claim made -through a view returns a remainder rooted in the global tokenizer. As a -result, `bindPhase()` can propagate the returned tokenizer directly and never -needs to: +A view changes visibility only. It never owns claims. Every claim made through a +view returns a remainder rooted in the global tokenizer. As a result, +`bindPhase()` can propagate the returned tokenizer directly and never needs to: 1. Materialize the visible tokens. 2. Compare them with a temporary remainder. 3. Recover the claimed indices. 4. Replay those claims against the global tokenizer. -The tokenizer should materialize one stable token array and carry one -cumulative immutable set of claimed indices. Claims should not create an -ever-deepening stack of tokenizer iterators. +The tokenizer should materialize one stable token array and carry one cumulative +immutable set of claimed indices. Claims should not create an ever-deepening +stack of tokenizer iterators. The remaining invariants are: @@ -218,7 +217,7 @@ existing segment start and end can therefore become a real range: ```ts interface TokenRange { readonly start: number; // exclusive - readonly end?: number; // exclusive + readonly end?: number; // exclusive } ``` @@ -274,11 +273,14 @@ function bindPhase(options: BindPhaseOptions): PhaseBinding { // Stable address sources: array order is precedence. for (let source of [fromValues, fromEnv]) { for (let param of pending.values()) { - accept(param, source({ + accept( param, - route: segment.path, - rest, - })); + source({ + param, + route: segment.path, + rest, + }), + ); } } @@ -305,8 +307,8 @@ no remaining words before or after it, the next comparison terminates. ## Parser integration -A phase owns the introduction of its value and environment sources; parser -state owns their lifetime after introduction. +A phase owns the introduction of its value and environment sources; parser state +owns their lifetime after introduction. When a phase becomes active, mount its newly declared sources at the current route path: @@ -319,10 +321,10 @@ rest = { }; ``` -Those sources remain available to every downstream phase. Input-level Values -and Env sources are mounted at `/` before the first phase. Sources introduced -on a child route are mounted at that child's route path. Sources introduced by -a dynamic continuation become visible only after that continuation is resumed. +Those sources remain available to every downstream phase. Input-level Values and +Env sources are mounted at `/` before the first phase. Sources introduced on a +child route are mounted at that child's route path. Sources introduced by a +dynamic continuation become visible only after that continuation is resumed. Parser state should carry `rest` as one value and replace it wholesale with the phase result: