diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0c3a89..6ae411e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,41 @@ jobs: git diff --exit-code spec/manifest.schema.json spec/auth-vectors.json || \ (echo "spec/manifest.schema.json or spec/auth-vectors.json is out of date — run 'pnpm gen:schema' and 'pnpm gen:auth-vectors' from ts/" && exit 1) + conformance: + name: Conformance (TS adapter) + # Runs the language-neutral conformance runner against the + # TypeScript SDK adapter. Issue #6 ships this as the reference; + # the Go adapter follows as a tracked follow-up. When the Go + # adapter lands, fan this out as a matrix entry per language. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.24" + cache: true + cache-dependency-path: go/go.sum + - uses: pnpm/action-setup@v4 + with: + package_json_file: ts/package.json + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: ts/pnpm-lock.yaml + - name: Install TS deps + working-directory: ts + run: pnpm install --frozen-lockfile + - name: Build SDK (produces dist/ for the adapter to import) + working-directory: ts/packages/sdk + run: pnpm build + - name: Run conformance vectors against the TypeScript adapter + working-directory: go + run: | + go run ./cmd/conformance \ + --vectors ../spec \ + --adapter "node ../ts/packages/sdk/test/conformance-adapter.mjs" + go: name: Go runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 57ecc3f..a714409 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ dist/ # Go go/cronix go/cronix-* +go/conformance *.exe *.test *.out diff --git a/go/cmd/conformance/main.go b/go/cmd/conformance/main.go new file mode 100644 index 0000000..29b41cc --- /dev/null +++ b/go/cmd/conformance/main.go @@ -0,0 +1,318 @@ +// Command conformance is the cronix language-neutral conformance runner. +// +// It reads spec/manifest-vectors.json + spec/auth-vectors.json and runs +// every vector against an "SDK adapter" — a small CLI any cronix SDK +// ships that responds to three subcommands via stdin/stdout: +// manifest-canonicalize, auth-sign, auth-verify. The contract is +// documented in spec/conformance/README.md. +// +// Usage: +// +// go run github.com/awbx/cronix/go/cmd/conformance \ +// --vectors spec \ +// --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" +// +// Exit codes: +// +// 0 every vector passed +// 1 at least one vector failed +// 2 configuration error (vectors missing, adapter not executable) +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" +) + +type manifestVectorFile struct { + Comment string `json:"$comment"` + Version int `json:"version"` + Vectors []manifestVector `json:"vectors"` +} + +type manifestVector struct { + Name string `json:"name"` + Valid bool `json:"valid"` + Input json.RawMessage `json:"input"` + Expected string `json:"expected"` + ErrorPaths []string `json:"errorPaths"` +} + +type authVectorFile struct { + Comment string `json:"$comment"` + Version int `json:"version"` + Vectors []authVector `json:"vectors"` +} + +type authVector struct { + Name string `json:"name"` + Kind string `json:"kind"` // "sign" or "verify" + + // sign + verify shared + Method string `json:"method"` + Path string `json:"path"` + BodyB64 string `json:"bodyB64"` + + // sign-specific + Secret string `json:"secret"` + Timestamp int64 `json:"timestamp"` + ExpectedHeader string `json:"expectedHeader"` + + // verify-specific + Secrets []string `json:"secrets"` + Header string `json:"header"` + Now int64 `json:"now"` + MaxSkewSeconds int `json:"maxSkewSeconds"` + Expect string `json:"expect"` // "ok" or an error code + ExpectedSecretIndex int `json:"expectedSecretIndex"` +} + +type result struct { + suite string + name string + passed bool + detail string +} + +func main() { + var ( + vectorsDir string + adapter string + ) + flag.StringVar(&vectorsDir, "vectors", "spec", "directory containing manifest-vectors.json and auth-vectors.json") + flag.StringVar(&adapter, "adapter", "", "adapter command (passed to sh -c so quoting and args work) — required") + flag.Parse() + + if adapter == "" { + fmt.Fprintln(os.Stderr, "error: --adapter is required") + os.Exit(2) + } + + manifestFile, err := loadManifestVectors(filepath.Join(vectorsDir, "manifest-vectors.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "error: load manifest vectors: %v\n", err) + os.Exit(2) + } + authFile, err := loadAuthVectors(filepath.Join(vectorsDir, "auth-vectors.json")) + if err != nil { + fmt.Fprintf(os.Stderr, "error: load auth vectors: %v\n", err) + os.Exit(2) + } + + var results []result + + for _, v := range manifestFile.Vectors { + results = append(results, runManifestVector(adapter, v)) + } + for _, v := range authFile.Vectors { + switch v.Kind { + case "sign": + results = append(results, runAuthSignVector(adapter, v)) + case "verify": + results = append(results, runAuthVerifyVector(adapter, v)) + default: + results = append(results, result{ + suite: "auth", + name: v.Name, + passed: false, + detail: fmt.Sprintf("unknown vector kind %q", v.Kind), + }) + } + } + + failed := 0 + for _, r := range results { + if r.passed { + fmt.Printf("PASS %-10s %s\n", r.suite, r.name) + } else { + fmt.Printf("FAIL %-10s %s\n %s\n", r.suite, r.name, r.detail) + failed++ + } + } + fmt.Printf("\n%d total / %d passed / %d failed\n", len(results), len(results)-failed, failed) + if failed > 0 { + os.Exit(1) + } +} + +func loadManifestVectors(path string) (*manifestVectorFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f manifestVectorFile + if err := json.Unmarshal(raw, &f); err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return &f, nil +} + +func loadAuthVectors(path string) (*authVectorFile, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f authVectorFile + if err := json.Unmarshal(raw, &f); err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + return &f, nil +} + +func runAdapter(adapter, subcommand string, input []byte) ([]byte, error) { + // Run via shell so the adapter flag can include arguments and quoting. + // Adapters are operator-supplied and run with full process privileges + // — same trust model as any other operator tool. Not for untrusted + // input. + cmd := exec.Command("sh", "-c", adapter+" "+subcommand) //#nosec G204 — operator-controlled, documented + cmd.Stdin = bytes.NewReader(input) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("adapter %s exited %v: stderr=%s", subcommand, err, strings.TrimSpace(stderr.String())) + } + return stdout.Bytes(), nil +} + +func runManifestVector(adapter string, v manifestVector) result { + r := result{suite: "manifest", name: v.Name} + out, err := runAdapter(adapter, "manifest-canonicalize", v.Input) + if err != nil { + r.detail = err.Error() + return r + } + outStr := strings.TrimRight(string(out), "\n") + + if v.Valid { + // Adapter must produce expected canonical JSON byte-for-byte. + if outStr == v.Expected { + r.passed = true + return r + } + // Tolerate object-key ordering differences by re-comparing parsed + // forms. If THAT also differs, it's a real divergence. + if jsonEquivalent(outStr, v.Expected) { + r.passed = true + r.detail = "(parsed-equivalent; byte-for-byte differs — see canonical-form discipline)" + return r + } + r.detail = fmt.Sprintf("expected:\n %s\ngot:\n %s", abbrev(v.Expected, 200), abbrev(outStr, 200)) + return r + } + + // Invalid vector: adapter must report at least one error path matching one of v.ErrorPaths. + var errOut struct { + Error struct { + Paths []string `json:"paths"` + } `json:"error"` + } + if err := json.Unmarshal([]byte(outStr), &errOut); err != nil { + r.detail = fmt.Sprintf("invalid vector but adapter did not emit {\"error\":{\"paths\":...}}: stdout=%s", abbrev(outStr, 200)) + return r + } + for _, want := range v.ErrorPaths { + if slices.Contains(errOut.Error.Paths, want) { + r.passed = true + return r + } + } + r.detail = fmt.Sprintf("expected at least one path from %v, got %v", v.ErrorPaths, errOut.Error.Paths) + return r +} + +func runAuthSignVector(adapter string, v authVector) result { + r := result{suite: "auth/sign", name: v.Name} + input, _ := json.Marshal(map[string]any{ + "secret": v.Secret, + "method": v.Method, + "path": v.Path, + "bodyB64": v.BodyB64, + "timestamp": v.Timestamp, + }) + out, err := runAdapter(adapter, "auth-sign", input) + if err != nil { + r.detail = err.Error() + return r + } + got := strings.TrimRight(string(out), "\n") + if got == v.ExpectedHeader { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected %q, got %q", v.ExpectedHeader, got) + return r +} + +func runAuthVerifyVector(adapter string, v authVector) result { + r := result{suite: "auth/verify", name: v.Name} + input, _ := json.Marshal(map[string]any{ + "secrets": v.Secrets, + "method": v.Method, + "path": v.Path, + "bodyB64": v.BodyB64, + "header": v.Header, + "now": v.Now, + "maxSkewSeconds": v.MaxSkewSeconds, + }) + out, err := runAdapter(adapter, "auth-verify", input) + if err != nil { + r.detail = err.Error() + return r + } + var resp struct { + OK bool `json:"ok"` + SecretIndex int `json:"secret_index"` + Error string `json:"error"` + } + if err := json.Unmarshal(out, &resp); err != nil { + r.detail = fmt.Sprintf("adapter response did not parse as JSON: %s", string(out)) + return r + } + if v.Expect == "ok" { + if resp.OK && resp.SecretIndex == v.ExpectedSecretIndex { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected ok+index=%d, got ok=%v index=%d error=%q", + v.ExpectedSecretIndex, resp.OK, resp.SecretIndex, resp.Error) + return r + } + if !resp.OK && resp.Error == v.Expect { + r.passed = true + return r + } + r.detail = fmt.Sprintf("expected error=%q, got ok=%v error=%q", v.Expect, resp.OK, resp.Error) + return r +} + +// jsonEquivalent compares two JSON strings as parsed values, ignoring key +// ordering. The canonical form should NOT differ in key order — if this +// path triggers a pass with a "byte-for-byte differs" note, the adapter +// likely has a canonicalization bug worth investigating. +func jsonEquivalent(a, b string) bool { + var av, bv any + if err := json.Unmarshal([]byte(a), &av); err != nil { + return false + } + if err := json.Unmarshal([]byte(b), &bv); err != nil { + return false + } + ab, _ := json.Marshal(av) + bb, _ := json.Marshal(bv) + return bytes.Equal(ab, bb) +} + +func abbrev(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/spec/conformance/README.md b/spec/conformance/README.md new file mode 100644 index 0000000..60ed1fd --- /dev/null +++ b/spec/conformance/README.md @@ -0,0 +1,78 @@ +# cronix conformance runner + +A language-neutral test harness that runs `spec/manifest-vectors.json` and `spec/auth-vectors.json` against an **SDK adapter** — a small CLI any cronix SDK ships so the runner can drive its library API through `stdin`/`stdout`. + +This directory is what makes the on-the-wire contract portable. Any future SDK (Rust, Python, Java, …) ships a conformance adapter and runs: + +```sh +go run github.com/awbx/cronix/go/cmd/conformance \ + --vectors spec \ + --adapter "" +``` + +If every vector passes, the SDK is byte-for-byte cronix-compliant. + +## The adapter contract + +An adapter is any command that responds to three subcommands. It reads a JSON object from stdin, writes a JSON object (or raw text where noted) to stdout, and exits 0 unconditionally — failures are reported via the JSON payload, not via exit code. **One adapter binary, three subcommands.** + +### ` manifest-canonicalize` + +Reads a manifest input. Produces the canonical normalized JSON form OR an error report. + +| Stream | Shape | +|---|---| +| stdin | A manifest input object (matches `spec/manifest.schema.json` for valid cases; arbitrary JSON for invalid cases) | +| stdout (valid) | The canonical normalized JSON string, with default policy fields filled in, jobs sorted, headers sorted, etc. — **byte-for-byte matching the vector's `expected` field** | +| stdout (invalid) | `{"error":{"paths":["a","b"]}}` listing JSON paths where validation failed | + +### ` auth-sign` + +Reads sign options. Produces the canonical HMAC-SHA256 header value. + +| Stream | Shape | +|---|---| +| stdin | `{"secret":"","method":"GET","path":"/api/x","bodyB64":"","timestamp":1730000000}` | +| stdout | The header string (raw text, no trailing newline). Format: `t=,v1=` | + +### ` auth-verify` + +Reads verify options. Reports whether the header is valid against the given secrets. + +| Stream | Shape | +|---|---| +| stdin | `{"secrets":["s1","s2"],"method":"GET","path":"/api/x","bodyB64":"","header":"
","now":1730000000,"maxSkewSeconds":300}` | +| stdout success | `{"ok":true,"secret_index":0}` — the index in `secrets` that produced a valid signature | +| stdout failure | `{"ok":false,"error":""}` — error codes per `spec/auth-vectors.json` (`"MalformedHeader"`, `"StaleTimestamp"`, `"SignatureMismatch"`) | + +## Running the runner + +```sh +# Against the bundled TypeScript adapter +go run ./go/cmd/conformance \ + --vectors spec \ + --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" + +# Against your own SDK adapter +go run ./go/cmd/conformance \ + --vectors spec \ + --adapter "./my-rust-adapter" +``` + +The runner prints per-vector pass/fail and exits non-zero if any vector fails. + +## Why a Go runner + +The runner has no SDK dependency — it shells out to adapters. It needs JSON parsing, subprocess management, and a CLI. Go provides all three with zero ceremony and is already in the cronix toolchain. **Adapter authors only need their own language's stdlib** to write the adapter; the runner runs everywhere Go runs. + +## Adding an adapter for a new SDK + +1. Implement the three subcommands above. The simplest adapter is ~50 lines of code calling your SDK's library API. +2. Run `go run ./go/cmd/conformance --adapter ""` against the bundled vectors. +3. When green, link your adapter from this README under `## Conformant adapters`. +4. (Optional) Open a PR adding your adapter as a CI matrix entry so cronix's CI catches future regressions in your SDK. + +## Conformant adapters + +- TypeScript (`@awbx/cronix-sdk`) — [`ts/packages/sdk/test/conformance-adapter.mjs`](../../ts/packages/sdk/test/conformance-adapter.mjs) +- Go (`pkg/cronsdk`) — *pending, tracked in a follow-up issue (see ROADMAP.md)* diff --git a/ts/packages/sdk/test/conformance-adapter.mjs b/ts/packages/sdk/test/conformance-adapter.mjs new file mode 100644 index 0000000..d7ab177 --- /dev/null +++ b/ts/packages/sdk/test/conformance-adapter.mjs @@ -0,0 +1,98 @@ +#!/usr/bin/env node +// cronix conformance adapter for @awbx/cronix-sdk. +// +// Implements the three-subcommand stdin/stdout contract documented in +// spec/conformance/README.md so the language-neutral runner +// (go/cmd/conformance) can drive this SDK and verify it agrees with +// every conformance vector byte-for-byte. +// +// Usage (from the repo root): +// +// go run ./go/cmd/conformance \ +// --vectors spec \ +// --adapter "node ts/packages/sdk/test/conformance-adapter.mjs" +// +// Build the SDK first if dist/ is stale: `pnpm -C ts/packages/sdk build`. + +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const distEntry = resolve(here, "../dist/index.js"); + +// Import from the built output rather than ts source so this adapter +// runs without ts-node. The CI step builds the SDK before invoking. +const { applyDefaults, canonicalize, parseManifest, sign, verify } = await import(distEntry); + +const subcommand = process.argv[2]; + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf8"); +} + +const fromB64 = (s) => (s === "" ? new Uint8Array(0) : Uint8Array.from(Buffer.from(s, "base64"))); + +function fail(msg) { + process.stderr.write(`adapter error: ${msg}\n`); + process.exit(2); +} + +async function main() { + const stdin = await readStdin(); + switch (subcommand) { + case "manifest-canonicalize": { + const input = JSON.parse(stdin); + const parsed = parseManifest(input); + if (!parsed.ok) { + // Per the contract, surface the JSON paths where validation + // failed. The TS SDK's issue.path is already an array of + // segments — join with "/" to match the vector format. + const paths = parsed.error.issues.map((i) => i.path.join("/")); + process.stdout.write(JSON.stringify({ error: { paths } })); + return; + } + const normalized = applyDefaults(parsed.value); + const canonical = canonicalize(normalized); + process.stdout.write(canonical); + return; + } + case "auth-sign": { + const opts = JSON.parse(stdin); + const result = await sign({ + secret: opts.secret, + method: opts.method, + path: opts.path, + body: fromB64(opts.bodyB64), + timestamp: opts.timestamp, + }); + process.stdout.write(result.header); + return; + } + case "auth-verify": { + const opts = JSON.parse(stdin); + const verifyOpts = { + secrets: opts.secrets, + method: opts.method, + path: opts.path, + body: fromB64(opts.bodyB64), + header: opts.header, + now: opts.now, + ...(opts.maxSkewSeconds !== undefined ? { maxSkewSeconds: opts.maxSkewSeconds } : {}), + }; + const result = await verify(verifyOpts); + if (result.ok) { + process.stdout.write(JSON.stringify({ ok: true, secret_index: result.value.secretIndex })); + } else { + process.stdout.write(JSON.stringify({ ok: false, error: result.error.code })); + } + return; + } + default: + fail(`unknown subcommand ${JSON.stringify(subcommand)}; expected one of manifest-canonicalize, auth-sign, auth-verify`); + } +} + +main().catch((err) => fail(err.stack ?? err.message ?? String(err)));