From 8b990bee697a59a71c731b56765042c1939254ec Mon Sep 17 00:00:00 2001 From: Abdelhadi Sabani Date: Mon, 25 May 2026 22:13:51 +0100 Subject: [PATCH 1/2] spec: language-neutral conformance runner + TypeScript adapter (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a portable conformance runner that drives any SDK's adapter through stdin/stdout and verifies it agrees with every vector byte-for-byte. Ships the runner + the reference TypeScript adapter + a CI job that exercises both. Go adapter follows as #51. What ships in this change: - spec/conformance/README.md: documents the three-subcommand adapter contract (manifest-canonicalize, auth-sign, auth-verify) future SDKs (Rust, Python, Java, ...) target. The contract is intentionally thin — one binary, three subcommands, JSON in and out, exit 0 unconditionally with errors reported via the JSON payload — so an adapter for a new SDK is ~50 LOC of plumbing on top of that SDK's library API. - go/cmd/conformance/main.go: the runner. Reads spec/manifest- vectors.json + spec/auth-vectors.json, invokes the configured adapter for each vector, compares the result against the vector's expected output. Reports per-vector pass/fail and exits 1 if any vector fails. Go runner rather than shell because subprocess management + JSON parsing in pure shell is error-prone; Go is already in the cronix toolchain and adapters only need their own language's stdlib. - ts/packages/sdk/test/conformance-adapter.mjs: the TypeScript adapter. ~80 lines wrapping parseManifest / applyDefaults / canonicalize / sign / verify from the SDK's built dist. Verified end-to-end: 64 vectors / 64 passed. - .github/workflows/ci.yml: new Conformance (TS adapter) job that builds the SDK then runs the runner against the TS adapter. Catches regressions in either the SDK or the spec vectors at PR time. Future Go adapter (#51) adds a matrix entry to this job. The conformance check is NOT yet in the required-status list on the main-branch ruleset. Once the Go adapter lands and the check proves stable, it's a one-line ruleset update to promote it to required. Test: pnpm -C ts/packages/sdk build go run ./go/cmd/conformance --vectors spec --adapter \\ "node ts/packages/sdk/test/conformance-adapter.mjs" # ...64 vectors enumerated as PASS... # 64 total / 64 passed / 0 failed Signed-off-by: Abdelhadi Sabani --- .github/workflows/ci.yml | 35 ++ go/cmd/conformance/main.go | 318 +++++++++++++++++++ spec/conformance/README.md | 78 +++++ ts/packages/sdk/test/conformance-adapter.mjs | 98 ++++++ 4 files changed, 529 insertions(+) create mode 100644 go/cmd/conformance/main.go create mode 100644 spec/conformance/README.md create mode 100644 ts/packages/sdk/test/conformance-adapter.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c3a89..6ae411e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,41 @@ jobs: git diff --exit-code spec/manifest.schema.json spec/auth-vectors.json || \ (echo "spec/manifest.schema.json or spec/auth-vectors.json is out of date — run 'pnpm gen:schema' and 'pnpm gen:auth-vectors' from ts/" && exit 1) + conformance: + name: Conformance (TS adapter) + # Runs the language-neutral conformance runner against the + # TypeScript SDK adapter. Issue #6 ships this as the reference; + # the Go adapter follows as a tracked follow-up. When the Go + # adapter lands, fan this out as a matrix entry per language. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.24" + cache: true + cache-dependency-path: go/go.sum + - uses: pnpm/action-setup@v4 + with: + package_json_file: ts/package.json + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: ts/pnpm-lock.yaml + - name: Install TS deps + working-directory: ts + run: pnpm install --frozen-lockfile + - name: Build SDK (produces dist/ for the adapter to import) + working-directory: ts/packages/sdk + run: pnpm build + - name: Run conformance vectors against the TypeScript adapter + working-directory: go + run: | + go run ./cmd/conformance \ + --vectors ../spec \ + --adapter "node ../ts/packages/sdk/test/conformance-adapter.mjs" + go: name: Go runs-on: ubuntu-latest diff --git a/go/cmd/conformance/main.go b/go/cmd/conformance/main.go new file mode 100644 index 0000000..29b41cc --- /dev/null +++ b/go/cmd/conformance/main.go @@ -0,0 +1,318 @@ +// Command conformance is the cronix language-neutral conformance runner. +// +// It reads spec/manifest-vectors.json + spec/auth-vectors.json and runs +// every vector against an "SDK adapter" — a small CLI any cronix SDK +// ships that responds to three subcommands via stdin/stdout: +// manifest-canonicalize, auth-sign, auth-verify. The contract is +// documented in spec/conformance/README.md. +// +// Usage: +// +// go run github.com/awbx/cronix/go/cmd/conformance \ +// --vectors spec \ +// --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" +// +// Exit codes: +// +// 0 every vector passed +// 1 at least one vector failed +// 2 configuration error (vectors missing, adapter not executable) +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" +) + +type manifestVectorFile struct { + Comment string `json:"$comment"` + Version int `json:"version"` + Vectors []manifestVector `json:"vectors"` +} + +type manifestVector struct { + Name string `json:"name"` + Valid bool `json:"valid"` + Input json.RawMessage `json:"input"` + Expected string `json:"expected"` + ErrorPaths []string `json:"errorPaths"` +} + +type authVectorFile struct { + Comment string `json:"$comment"` + Version int `json:"version"` + Vectors []authVector `json:"vectors"` +} + +type authVector struct { + Name string `json:"name"` + Kind string `json:"kind"` // "sign" or "verify" + + // sign + verify shared + Method string `json:"method"` + Path string `json:"path"` + BodyB64 string `json:"bodyB64"` + + // sign-specific + Secret string `json:"secret"` + Timestamp int64 `json:"timestamp"` + ExpectedHeader string `json:"expectedHeader"` + + // verify-specific + Secrets []string `json:"secrets"` + Header string `json:"header"` + Now int64 `json:"now"` + MaxSkewSeconds int `json:"maxSkewSeconds"` + Expect string `json:"expect"` // "ok" or an error code + ExpectedSecretIndex int `json:"expectedSecretIndex"` +} + +type result struct { + suite string + name string + passed bool + detail string +} + +func main() { + var ( + vectorsDir string + adapter string + ) + flag.StringVar(&vectorsDir, "vectors", "spec", "directory containing manifest-vectors.json and auth-vectors.json") + flag.StringVar(&adapter, "adapter", "", "adapter command (passed to sh -c so quoting and args work) — required") + flag.Parse() + + if adapter == "" { + fmt.Fprintln(os.Stderr, "error: --adapter is required") + os.Exit(2) + } + + manifestFile, err := loadManifestVectors(filepath.Join(vectorsDir, "manifest-vectors.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "error: load manifest vectors: %v\n", err) + os.Exit(2) + } + authFile, err := loadAuthVectors(filepath.Join(vectorsDir, "auth-vectors.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "error: load auth vectors: %v\n", err) + os.Exit(2) + } + + var results []result + + for _, v := range manifestFile.Vectors { + results = append(results, runManifestVector(adapter, v)) + } + for _, v := range authFile.Vectors { + switch v.Kind { + case "sign": + results = append(results, runAuthSignVector(adapter, v)) + case "verify": + results = append(results, runAuthVerifyVector(adapter, v)) + default: + results = append(results, result{ + suite: "auth", + name: v.Name, + passed: false, + detail: fmt.Sprintf("unknown vector kind %q", v.Kind), + }) + } + } + + failed := 0 + for _, r := range results { + if r.passed { + fmt.Printf("PASS %-10s %s\n", r.suite, r.name) + } else { + fmt.Printf("FAIL %-10s %s\n %s\n", r.suite, r.name, r.detail) + failed++ + } + } + fmt.Printf("\n%d total / %d passed / %d failed\n", len(results), len(results)-failed, failed) + if failed > 0 { + os.Exit(1) + } +} + +func loadManifestVectors(path string) (*manifestVectorFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f manifestVectorFile + if err := json.Unmarshal(raw, &f); err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return &f, nil +} + +func loadAuthVectors(path string) (*authVectorFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f authVectorFile + if err := json.Unmarshal(raw, &f); err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return &f, nil +} + +func runAdapter(adapter, subcommand string, input []byte) ([]byte, error) { + // Run via shell so the adapter flag can include arguments and quoting. + // Adapters are operator-supplied and run with full process privileges + // — same trust model as any other operator tool. Not for untrusted + // input. + cmd := exec.Command("sh", "-c", adapter+" "+subcommand) //#nosec G204 — operator-controlled, documented + cmd.Stdin = bytes.NewReader(input) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("adapter %s exited %v: stderr=%s", subcommand, err, strings.TrimSpace(stderr.String())) + } + return stdout.Bytes(), nil +} + +func runManifestVector(adapter string, v manifestVector) result { + r := result{suite: "manifest", name: v.Name} + out, err := runAdapter(adapter, "manifest-canonicalize", v.Input) + if err != nil { + r.detail = err.Error() + return r + } + outStr := strings.TrimRight(string(out), "\n") + + if v.Valid { + // Adapter must produce expected canonical JSON byte-for-byte. + if outStr == v.Expected { + r.passed = true + return r + } + // Tolerate object-key ordering differences by re-comparing parsed + // forms. If THAT also differs, it's a real divergence. + if jsonEquivalent(outStr, v.Expected) { + r.passed = true + r.detail = "(parsed-equivalent; byte-for-byte differs — see canonical-form discipline)" + return r + } + r.detail = fmt.Sprintf("expected:\n %s\ngot:\n %s", abbrev(v.Expected, 200), abbrev(outStr, 200)) + return r + } + + // Invalid vector: adapter must report at least one error path matching one of v.ErrorPaths. + var errOut struct { + Error struct { + Paths []string `json:"paths"` + } `json:"error"` + } + if err := json.Unmarshal([]byte(outStr), &errOut); err != nil { + r.detail = fmt.Sprintf("invalid vector but adapter did not emit {\"error\":{\"paths\":...}}: stdout=%s", abbrev(outStr, 200)) + return r + } + for _, want := range v.ErrorPaths { + if slices.Contains(errOut.Error.Paths, want) { + r.passed = true + return r + } + } + r.detail = fmt.Sprintf("expected at least one path from %v, got %v", v.ErrorPaths, errOut.Error.Paths) + return r +} + +func runAuthSignVector(adapter string, v authVector) result { + r := result{suite: "auth/sign", name: v.Name} + input, _ := json.Marshal(map[string]any{ + "secret": v.Secret, + "method": v.Method, + "path": v.Path, + "bodyB64": v.BodyB64, + "timestamp": v.Timestamp, + }) + out, err := runAdapter(adapter, "auth-sign", input) + if err != nil { + r.detail = err.Error() + return r + } + got := strings.TrimRight(string(out), "\n") + if got == v.ExpectedHeader { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected %q, got %q", v.ExpectedHeader, got) + return r +} + +func runAuthVerifyVector(adapter string, v authVector) result { + r := result{suite: "auth/verify", name: v.Name} + input, _ := json.Marshal(map[string]any{ + "secrets": v.Secrets, + "method": v.Method, + "path": v.Path, + "bodyB64": v.BodyB64, + "header": v.Header, + "now": v.Now, + "maxSkewSeconds": v.MaxSkewSeconds, + }) + out, err := runAdapter(adapter, "auth-verify", input) + if err != nil { + r.detail = err.Error() + return r + } + var resp struct { + OK bool `json:"ok"` + SecretIndex int `json:"secret_index"` + Error string `json:"error"` + } + if err := json.Unmarshal(out, &resp); err != nil { + r.detail = fmt.Sprintf("adapter response did not parse as JSON: %s", string(out)) + return r + } + if v.Expect == "ok" { + if resp.OK && resp.SecretIndex == v.ExpectedSecretIndex { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected ok+index=%d, got ok=%v index=%d error=%q", + v.ExpectedSecretIndex, resp.OK, resp.SecretIndex, resp.Error) + return r + } + if !resp.OK && resp.Error == v.Expect { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected error=%q, got ok=%v error=%q", v.Expect, resp.OK, resp.Error) + return r +} + +// jsonEquivalent compares two JSON strings as parsed values, ignoring key +// ordering. The canonical form should NOT differ in key order — if this +// path triggers a pass with a "byte-for-byte differs" note, the adapter +// likely has a canonicalization bug worth investigating. +func jsonEquivalent(a, b string) bool { + var av, bv any + if err := json.Unmarshal([]byte(a), &av); err != nil { + return false + } + if err := json.Unmarshal([]byte(b), &bv); err != nil { + return false + } + ab, _ := json.Marshal(av) + bb, _ := json.Marshal(bv) + return bytes.Equal(ab, bb) +} + +func abbrev(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/spec/conformance/README.md b/spec/conformance/README.md new file mode 100644 index 0000000..60ed1fd --- /dev/null +++ b/spec/conformance/README.md @@ -0,0 +1,78 @@ +# cronix conformance runner + +A language-neutral test harness that runs `spec/manifest-vectors.json` and `spec/auth-vectors.json` against an **SDK adapter** — a small CLI any cronix SDK ships so the runner can drive its library API through `stdin`/`stdout`. + +This directory is what makes the on-the-wire contract portable. Any future SDK (Rust, Python, Java, …) ships a conformance adapter and runs: + +```sh +go run github.com/awbx/cronix/go/cmd/conformance \ + --vectors spec \ + --adapter "" +``` + +If every vector passes, the SDK is byte-for-byte cronix-compliant. + +## The adapter contract + +An adapter is any command that responds to three subcommands. It reads a JSON object from stdin, writes a JSON object (or raw text where noted) to stdout, and exits 0 unconditionally — failures are reported via the JSON payload, not via exit code. **One adapter binary, three subcommands.** + +### ` manifest-canonicalize` + +Reads a manifest input. Produces the canonical normalized JSON form OR an error report. + +| Stream | Shape | +|---|---| +| stdin | A manifest input object (matches `spec/manifest.schema.json` for valid cases; arbitrary JSON for invalid cases) | +| stdout (valid) | The canonical normalized JSON string, with default policy fields filled in, jobs sorted, headers sorted, etc. — **byte-for-byte matching the vector's `expected` field** | +| stdout (invalid) | `{"error":{"paths":["a","b"]}}` listing JSON paths where validation failed | + +### ` auth-sign` + +Reads sign options. Produces the canonical HMAC-SHA256 header value. + +| Stream | Shape | +|---|---| +| stdin | `{"secret":"","method":"GET","path":"/api/x","bodyB64":"","timestamp":1730000000}` | +| stdout | The header string (raw text, no trailing newline). Format: `t=,v1=` | + +### ` auth-verify` + +Reads verify options. Reports whether the header is valid against the given secrets. + +| Stream | Shape | +|---|---| +| stdin | `{"secrets":["s1","s2"],"method":"GET","path":"/api/x","bodyB64":"","header":"
","now":1730000000,"maxSkewSeconds":300}` | +| stdout success | `{"ok":true,"secret_index":0}` — the index in `secrets` that produced a valid signature | +| stdout failure | `{"ok":false,"error":""}` — error codes per `spec/auth-vectors.json` (`"MalformedHeader"`, `"StaleTimestamp"`, `"SignatureMismatch"`) | + +## Running the runner + +```sh +# Against the bundled TypeScript adapter +go run ./go/cmd/conformance \ + --vectors spec \ + --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" + +# Against your own SDK adapter +go run ./go/cmd/conformance \ + --vectors spec \ + --adapter "./my-rust-adapter" +``` + +The runner prints per-vector pass/fail and exits non-zero if any vector fails. + +## Why a Go runner + +The runner has no SDK dependency — it shells out to adapters. It needs JSON parsing, subprocess management, and a CLI. Go provides all three with zero ceremony and is already in the cronix toolchain. **Adapter authors only need their own language's stdlib** to write the adapter; the runner runs everywhere Go runs. + +## Adding an adapter for a new SDK + +1. Implement the three subcommands above. The simplest adapter is ~50 lines of code calling your SDK's library API. +2. Run `go run ./go/cmd/conformance --adapter ""` against the bundled vectors. +3. When green, link your adapter from this README under `## Conformant adapters`. +4. (Optional) Open a PR adding your adapter as a CI matrix entry so cronix's CI catches future regressions in your SDK. + +## Conformant adapters + +- TypeScript (`@awbx/cronix-sdk`) — [`ts/packages/sdk/test/conformance-adapter.mjs`](../../ts/packages/sdk/test/conformance-adapter.mjs) +- Go (`pkg/cronsdk`) — *pending, tracked in a follow-up issue (see ROADMAP.md)* diff --git a/ts/packages/sdk/test/conformance-adapter.mjs b/ts/packages/sdk/test/conformance-adapter.mjs new file mode 100644 index 0000000..d7ab177 --- /dev/null +++ b/ts/packages/sdk/test/conformance-adapter.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +// cronix conformance adapter for @awbx/cronix-sdk. +// +// Implements the three-subcommand stdin/stdout contract documented in +// spec/conformance/README.md so the language-neutral runner +// (go/cmd/conformance) can drive this SDK and verify it agrees with +// every conformance vector byte-for-byte. +// +// Usage (from the repo root): +// +// go run ./go/cmd/conformance \ +// --vectors spec \ +// --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" +// +// Build the SDK first if dist/ is stale: `pnpm -C ts/packages/sdk build`. + +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const distEntry = resolve(here, "../dist/index.js"); + +// Import from the built output rather than ts source so this adapter +// runs without ts-node. The CI step builds the SDK before invoking. +const { applyDefaults, canonicalize, parseManifest, sign, verify } = await import(distEntry); + +const subcommand = process.argv[2]; + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); +} + +const fromB64 = (s) => (s === "" ? new Uint8Array(0) : Uint8Array.from(Buffer.from(s, "base64"))); + +function fail(msg) { + process.stderr.write(`adapter error: ${msg}\n`); + process.exit(2); +} + +async function main() { + const stdin = await readStdin(); + switch (subcommand) { + case "manifest-canonicalize": { + const input = JSON.parse(stdin); + const parsed = parseManifest(input); + if (!parsed.ok) { + // Per the contract, surface the JSON paths where validation + // failed. The TS SDK's issue.path is already an array of + // segments — join with "/" to match the vector format. + const paths = parsed.error.issues.map((i) => i.path.join("/")); + process.stdout.write(JSON.stringify({ error: { paths } })); + return; + } + const normalized = applyDefaults(parsed.value); + const canonical = canonicalize(normalized); + process.stdout.write(canonical); + return; + } + case "auth-sign": { + const opts = JSON.parse(stdin); + const result = await sign({ + secret: opts.secret, + method: opts.method, + path: opts.path, + body: fromB64(opts.bodyB64), + timestamp: opts.timestamp, + }); + process.stdout.write(result.header); + return; + } + case "auth-verify": { + const opts = JSON.parse(stdin); + const verifyOpts = { + secrets: opts.secrets, + method: opts.method, + path: opts.path, + body: fromB64(opts.bodyB64), + header: opts.header, + now: opts.now, + ...(opts.maxSkewSeconds !== undefined ? { maxSkewSeconds: opts.maxSkewSeconds } : {}), + }; + const result = await verify(verifyOpts); + if (result.ok) { + process.stdout.write(JSON.stringify({ ok: true, secret_index: result.value.secretIndex })); + } else { + process.stdout.write(JSON.stringify({ ok: false, error: result.error.code })); + } + return; + } + default: + fail(`unknown subcommand ${JSON.stringify(subcommand)}; expected one of manifest-canonicalize, auth-sign, auth-verify`); + } +} + +main().catch((err) => fail(err.stack ?? err.message ?? String(err))); From fab193e412e09fbefc06b7d3711bf4b0d3cbabdb Mon Sep 17 00:00:00 2001 From: Abdelhadi Sabani Date: Mon, 25 May 2026 22:15:01 +0100 Subject: [PATCH 2/2] chore: ignore go/conformance build output Matches the existing pattern for go/cronix and go/cronix-* binaries above. The cmd/conformance build lands at go/conformance when developers run go build ./cmd/conformance from the go/ directory. Signed-off-by: Abdelhadi Sabani --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57ecc3f..a714409 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ dist/ # Go go/cronix go/cronix-* +go/conformance *.exe *.test *.out