diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca1a7471..fc630e86 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -86,9 +86,7 @@ jobs:
run: python3 eng/verify-vault-publish-boundary.py
- name: Test local decision service and evaluation tools
- run: |
- python3 -B -m unittest discover -s tests/laya-service -v
- python3 -B -m unittest discover -s tests/routing-eval -p test_jev_report.py
+ run: dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --no-build
- name: Test standard targets
run: dotnet test --no-build -c Release --verbosity normal --logger "trx;LogFileName=results.trx" src/OpenClaw.Tests
diff --git a/OpenClaw.Net.slnx b/OpenClaw.Net.slnx
index 35c6bb18..35b3a02b 100644
--- a/OpenClaw.Net.slnx
+++ b/OpenClaw.Net.slnx
@@ -55,6 +55,13 @@
+
+
+
+
+
+
+
diff --git a/docs/README.md b/docs/README.md
index d8489847..400142ce 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -43,6 +43,7 @@ Use this page as the map. If you are unsure where to go next, the groups below a
| [opensquilla-meta-skill-migration.md](opensquilla-meta-skill-migration.md) | OpenClaw.NET meta-skill migration notes for OpenSquilla-style DAG orchestration, current parity, and remaining migration gaps. |
| [dynamic-turn-routing-model-profiles.md](dynamic-turn-routing-model-profiles.md) | Collaboration playbook for how Dynamic Turn Routing and Model Profiles work together (precedence, field projection, fallback, and restore semantics). |
| [laya-routing.md](laya-routing.md) | Local Laya decision routing, model preparation, calibration, attribution, and rollback. |
+| [zh-CN/integrations/laya-routing.md](zh-CN/integrations/laya-routing.md) | 本地 Laya 路由服务、.NET CLI、模型校验、校准与回滚（简体中文）。 |
| [jev-routing.md](jev-routing.md) | Optional Jev shadow evaluation and model routing, configuration, diagnostics, quality reports, and rollback. |
| [EXTERNAL_CLI_CONNECTORS.md](EXTERNAL_CLI_CONNECTORS.md) | Governed external CLI connectors, optional presets, named command allowlists, approvals, redaction, and audit behavior. |
| [plugins/payment.md](plugins/payment.md) | Native payment tool, virtual cards, machine payments, providers, and safe agent-facing actions. |
diff --git a/docs/SITE_MAP.md b/docs/SITE_MAP.md
index ffd3b227..db1d786e 100644
--- a/docs/SITE_MAP.md
+++ b/docs/SITE_MAP.md
@@ -56,6 +56,9 @@ Use this map when turning the Markdown docs into a documentation website. It kee
| Integrations | Microsoft Teams | [TEAMS_SETUP.md](TEAMS_SETUP.md) |
| Integrations | WhatsApp | [WHATSAPP_SETUP.md](WHATSAPP_SETUP.md) |
| Integrations | A2A | [a2a.md](a2a.md) |
+| Integrations | Local Laya Decision Routing | [laya-routing.md](laya-routing.md) |
+| Integrations | Jev Decision Routing | [jev-routing.md](jev-routing.md) |
+| Integrations | Local Laya Routing (zh-CN) | [zh-CN/integrations/laya-routing.md](zh-CN/integrations/laya-routing.md) |
| Integrations | External Coding Backends | [external-coding-backends.md](external-coding-backends.md) |
| Integrations | Nacos MCP Router (PoC) | [nacos-mcp-router.md](nacos-mcp-router.md) |
| Integrations | Nacos MCP Router (PoC, zh-CN) | [zh-CN/nacos-mcp-router.md](zh-CN/nacos-mcp-router.md) |
@@ -146,6 +149,9 @@ Integrations
Microsoft Teams
WhatsApp
A2A
+ Local Laya Decision Routing
+ Jev Decision Routing
+ Local Laya Routing (zh-CN)
External Coding Backends
Nacos MCP Router (PoC)
Tailscale Deployment
diff --git a/docs/cli/routing.md b/docs/cli/routing.md
index 8068e68f..b897de8b 100644
--- a/docs/cli/routing.md
+++ b/docs/cli/routing.md
@@ -27,6 +27,14 @@ Router mode behavior:
`routing status` also reports `jevMode`, `jevModel`, `layaMode`, `layaModel`, and `layaCalibrationId`. See [local Laya routing](../laya-routing.md) for offline setup and calibration. Jev is disabled by default; its `shadow` and `active` modes are configured through `DynamicTurnRouting.Jev`. See [Jev routing](../jev-routing.md) for the evaluation workflow. The existing `diagnostics on|off` command controls the ONNX policy flag; Jev journaling is controlled separately by `Jev.DiagnosticsPath`.
+Jev and Laya journal summaries share the standalone .NET report tool. It can write JSON and optional ScottPlot PNG reliability/risk-coverage charts; see the provider guides for label schema and examples:
+
+```bash
+dotnet run --project tools/laya_service -c Release -- report \
+ /path/to/decisions.snapshot.jsonl --labels /path/to/labels.jsonl \
+ --output /path/to/report.json --plot /path/to/reliability.png
+```
+
Examples:
```bash
diff --git a/docs/jev-routing.md b/docs/jev-routing.md
index 2352fa4b..99ac94a6 100644
--- a/docs/jev-routing.md
+++ b/docs/jev-routing.md
@@ -62,7 +62,8 @@ The tier/profile fields describe routing preferences before the existing model s
Copy a completed journal snapshot and summarize it:
```bash
-python3 scripts/evaluate-jev-routing.py /path/to/jev-decisions.snapshot.jsonl --output /tmp/jev-report.json
+dotnet run --project tools/laya_service -c Release -- report \
+ /path/to/jev-decisions.snapshot.jsonl --output /tmp/jev-report.json
```
For representative decisions, use existing session records to assign human labels in a separate JSONL file:
@@ -72,7 +73,9 @@ For representative decisions, use existing session records to assign human label
```
```bash
-python3 scripts/evaluate-jev-routing.py /path/to/jev-decisions.snapshot.jsonl --labels /path/to/labels.jsonl --output /tmp/jev-quality.json
+dotnet run --project tools/laya_service -c Release -- report \
+ /path/to/jev-decisions.snapshot.jsonl --labels /path/to/labels.jsonl \
+ --output /tmp/jev-quality.json
```
Without labels the report includes coverage, failures, disagreement, latency and estimated decision spend; it makes no accuracy claim. With labels it compares the baseline, always-T2, and Jev with safeguards/fallback using accuracy, confusion matrices, per-tier F1, under-routing, and high-risk capability retention. Label coverage and per-tier sample counts matter. Separate model/rubric cohorts before calibrating thresholds.
@@ -104,7 +107,7 @@ Restart afterward and remove any environment override that would re-enable Jev.
```bash
dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -p:OpenClawSkipDashboardBuild=true --filter 'FullyQualifiedName~JevRoutingTests|FullyQualifiedName~TurnRoutingPolicyTests|FullyQualifiedName~RoutingCommandsTests'
-python3 -m unittest discover -s tests/routing-eval -p test_jev_report.py
+dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~RoutingJournalReportTests|FullyQualifiedName~ReliabilityPlotTests'
```
The transport follows the [TypeSafe API](https://docs.typesafe.ai/api). The model pin and initial estimated price come from its [model reference](https://docs.typesafe.ai/models), checked September 20, 2026. See [confidence semantics](https://docs.typesafe.ai/confidence) and [known model limitations](https://docs.typesafe.ai/model-jaggedness/jev-1.13) before tuning policy.
diff --git a/docs/laya-routing.md b/docs/laya-routing.md
index d4ba9b1c..23ecf7b7 100644
--- a/docs/laya-routing.md
+++ b/docs/laya-routing.md
@@ -1,41 +1,38 @@
-# Local Laya decision routing
+# Local Laya Decision Routing
-OpenClaw can use Laya as an optional, locally hosted decision provider. Both native and Microsoft Agent Framework runtimes use the shared decision-routing policy. The default is disabled. Laya produces typed decisions; it is not a chat model and should not be added to `Models.Profiles` as a generator.
+OpenClaw can use Laya as an optional, locally hosted decision provider. Native and Microsoft Agent Framework runtimes share the decision-routing policy. The default is disabled. Laya produces typed decisions; it is not a chat model and must not be added to `Models.Profiles` as a generator.
-**Laya is developed by Nandakishor Mukkunnoth (Nandakishor M), ConvAI Innovations, and upstream contributors.** Its model, SDK, and research are their work. This integration's local service, compatibility checks, and evaluation tooling are maintained separately. See the [author's article](https://laya.convaiinnovations.com/), [source repository](https://github.com/NandhaKishorM/laya), [model card](https://huggingface.co/convaiinnovations/laya), and retained [attribution and license](../tools/laya_service/THIRD_PARTY_NOTICES.md). The earlier [confidence-aware routing paper](https://arxiv.org/abs/2510.01237) motivates escalation pathways; this implementation changes model selection only.
+**Laya is developed by Nandakishor Mukkunnoth (Nandakishor M), ConvAI Innovations, and upstream contributors.** Its model, SDK, and research are their work. OpenClaw separately maintains the local service, adapter, and evaluation tools. See the [author's article](https://laya.convaiinnovations.com/), [source repository](https://github.com/NandhaKishorM/laya), [model card](https://huggingface.co/convaiinnovations/laya), and retained [attribution and license](../tools/laya_service/THIRD_PARTY_NOTICES.md). The earlier [confidence-aware routing paper](https://arxiv.org/abs/2510.01237) motivates escalation pathways; this integration changes model selection only.
-## Prepare the optional service
+## Runtime Boundary
-Run these commands from the repository root with Python 3.12. Python/PyTorch remain outside the .NET gateway and its NativeAOT binary. Keep model assets and calibration datasets outside the repository.
+The standalone service targets .NET 10 and uses `NLaya`/`NLaya.TorchSharp` 1.0.0 with `TorchSharp-cpu` 0.107.0. The Gateway communicates with it over the loopback HTTP contract and does not reference NLaya or TorchSharp. The service is a JIT deployment; NativeAOT support is not claimed. CPU is the verified runtime with the checked-in dependency set. `cuda` and `mps` are accepted only when their matching TorchSharp backend is installed; this project does not bundle those backends.
+
+Keep model assets, calibration observations, and reports outside the repository. From the repository root, download the pinned revision and selected checkpoint files:
```bash
-python3.12 -m venv /path/to/laya-venv
-/path/to/laya-venv/bin/python -m pip install -r tools/laya_service/requirements.txt
-/path/to/laya-venv/bin/python -m tools.laya_service.download \
+dotnet run --project tools/laya_service -c Release -- download \
--destination /path/to/laya-models \
+ --revision 1c5edc17a7acd8701df6fc341c0d179f1c62c982 \
--checkpoint english --checkpoint multilingual
```
-The downloader pins Hugging Face revision `1c5edc17a7acd8701df6fc341c0d179f1c62c982`, selects only the named checkpoint files, prepares tokenizer compatibility, and creates a manifest of SHA-256 hashes. It copies attribution and the upstream Apache-2.0 license beside the assets. Use a new destination when changing revision. The SDK is pinned to `laya==0.3.4`; dependency versions are the tested Python 3.12 combination. On Linux CUDA hosts, install the appropriate PyTorch wheel for the GPU/driver. CUDA was not verified by the macOS smoke test.
-
-For English-only use, download only `english`. Non-English requests then retain the gateway's baseline when the required checkpoint is unavailable. Do not assume unknown or mixed scripts are English. The compatibility layer handles Armenian, omitted by the released SDK, and other non-Latin minority scripts before inference. An explicit language hint helps short Latin-script inputs; an English hint conflicting with non-Latin text is rejected.
+The downloader retrieves only the allowlisted model/config/tokenizer assets for the requested checkpoints, writes their SHA-256 values to `manifest.json`, and copies upstream attribution and license notices. The default revision is `1c5edc17a7acd8701df6fc341c0d179f1c62c982`. Use a fresh destination when changing revisions. Startup verifies manifest paths and every asset hash before loading local files; serving never downloads weights. The [retained upstream license](../tools/laya_service/licenses/laya-APACHE-2.0.txt) and [third-party notices](../tools/laya_service/THIRD_PARTY_NOTICES.md) travel with the service.
-Start a persistent service:
+Start the local service:
```bash
-/path/to/laya-venv/bin/python -m tools.laya_service \
- --manifest /path/to/laya-models/manifest.json --device cpu --port 8099
+dotnet run --project tools/laya_service -c Release -- serve \
+ --manifest /path/to/laya-models/manifest.json --device cpu --port 8099 --threads 4
```
-Use `--device mps` on supported Apple Silicon or `--device cuda` on a configured NVIDIA host. `--threads 4` is the default CPU thread limit. `--checkpoint multilingual` can force all requests to the multilingual checkpoint. The workflow-specific `typed-decisions` checkpoint must be downloaded and explicitly selected; automatic routing does not select it merely because question names resemble its training tasks.
-
-All installed/selected checkpoints load and warm up before the socket opens. Startup verifies every asset hash. Runtime sets Hugging Face/Transformers offline mode and loads only local paths; it never downloads weights. `GET http://127.0.0.1:8099/health` reports readiness, model identity, calibration ID, checkpoints, and actual devices, including SDK CPU fallback. Restart the service after changing its manifest or calibration.
+`--checkpoint auto` is the default. `english` and `multilingual` are routed by language; `typed-decisions` must be downloaded and explicitly selected. `--device` accepts `cpu`, `cuda`, or `mps`; an unavailable backend fails startup rather than silently falling back. `GET http://127.0.0.1:8099/health` reports readiness and model/runtime identity. Restart after changing the manifest or calibration.
-The service binds only to `127.0.0.1`, rejects browser Origin and unexpected Host headers, caps request size and connections, and admits one inference at a time without a waiting queue. Protect the host as you would any local process. The gateway bypasses proxies and redirects for Laya, accepts only literal loopback endpoints, and never falls back from Laya to hosted Jev. Selected conversation text still crosses a local HTTP process boundary.
+The service binds only to `127.0.0.1`, rejects browser Origin and unexpected Host headers, caps request size and connections, and admits one inference at a time without a waiting queue. The Gateway bypasses proxies and redirects, accepts only literal loopback endpoints, and never falls back from Laya to hosted Jev. Selected conversation text still crosses a local HTTP process boundary.
-## Start in shadow mode
+## Gateway Configuration
-Retain the existing `Policy.Tiers` model-profile mappings. Enable one decision provider at a time:
+Retain existing `Policy.Tiers` model-profile mappings and enable one decision provider at a time:
```json
{
@@ -57,57 +54,66 @@ Retain the existing `Policy.Tiers` model-profile mappings. Enable one decision p
}
```
-Restart the gateway. `openclaw routing status --config /path/to/appsettings.json` shows configured provider modes and identities. `DynamicTurnRouting.Enabled` still controls the independent ONNX baseline. Shadow mode returns that baseline unchanged but adds bounded evaluation latency.
+Restart the Gateway. `openclaw routing status --config /path/to/appsettings.json` shows configured provider modes and identities. `DynamicTurnRouting.Enabled` still controls the independent ONNX baseline. Shadow mode returns that baseline unchanged but adds bounded evaluation latency.
-The shorter `openclaw-laya-tiers-v1` rubric is [shared by the gateway and evaluation tool](../tools/laya_service/rubrics/openclaw-laya-tiers-v1.json). It asks for T0–T3/abstain, risk, and tool need. Before prediction the service checks the exact tokenizer budget for state, question instructions, and every option. It rejects any input the SDK would truncate or rewrite, rather than deciding from silently incomplete text. The English checkpoint has a 512-token total budget per question, multilingual 1,024 by default, including questions and options. Up to 20 choice options and 16 questions are accepted, subject to those stricter token limits.
+The `openclaw-laya-tiers-v1` rubric is [shared by the Gateway and evaluator](../tools/laya_service/rubrics/openclaw-laya-tiers-v1.json). The adapter reports `sdk_version: "1.0.0"` and `runtime: "NLaya"`; Gateway metadata checks both before accepting a decision. The service checks tokenizer budgets for state, question instructions, and options, and rejects inputs that would be truncated. The English checkpoint budget is 512 tokens per question and multilingual is 1,024 by default; protocol limits are up to 20 choice options and 16 questions, subject to those budgets.
-The adapter checks model revision, SDK version, calibration identity, rubric, device, checkpoint, and explicit absence of truncation. Errors, overload, deadline expiry, and uncertainty retain the baseline. The shared policy preserves redaction, explicit model selections, deterministic floors, sticky tiers, and tool permissions. It does not authorize actions. Disabling a router does not weaken existing permissions.
+Errors, overload, deadline expiry, and uncertainty retain the baseline. Shared policy preserves redaction, explicit model selections, deterministic floors, sticky tiers, and tool permissions. It does not authorize actions. Metadata-only journals omit state, prompts, and credentials; protect and rotate them using normal log retention controls.
-Metadata-only journals include provider, checkpoint/revision, schema hash, calibration ID, actual device, rubric, probabilities, latency, and applied/proposed routes. They omit state, prompts, and credentials. Laya's metered API cost is zero; this does not represent the cost of local compute. Rotate/protect these files using normal log retention controls.
+## Evaluate and Calibrate
-## Evaluate and calibrate
-
-The base weights run without training, but this is not evidence of production routing quality. In a local smoke test the new rubric still underpredicted tool need for repository work. The [author's benchmarks](https://github.com/NandhaKishorM/laya/blob/42626c348753fbb17572a813127df2278a1ec527/BENCHMARKS.md) also distinguish the fine-tuned checkpoint from weak base-model performance on typed decisions. Do not transfer Jev confidence thresholds or advertise the local timing as a quality benchmark.
-
-Prepare representative, human-labeled calibration and held-out validation JSONL files. Each line has a unique case ID, state, and labels for every question. The tool supplies the shared routing rubric and pinned model unless an explicit schema/model is provided:
+Model execution and passing unit tests do not establish production routing quality. Do not transfer Jev confidence thresholds or interpret local runtime as a quality benchmark. Prepare representative human-labeled calibration and held-out validation JSONL files. Each case has a unique ID, state, and labels for every question; when questions/model are omitted, the evaluator supplies the shared rubric and pinned model:
```json
{"case_id":"cal-001","state":{"current_request":"Rewrite hello in uppercase.","recent_conversation":[]},"labels":{"tier":"T0","high_risk":false,"requires_tools":false}}
```
-Include languages, multi-turn references, ambiguity, consequential tasks, and tool-dependent requests. Keep validation cases and their IDs separate from fitting data. Twenty observations per checkpoint/question-type/option-count bucket is the tooling minimum, not a sufficient production sample size. Include multiple labels in each calibration bucket. Do not derive tool/risk labels mechanically from model outputs.
+Include languages, multi-turn references, ambiguity, consequential tasks, and tool-dependent requests. Keep validation cases separate from fitting cases. Twenty observations per checkpoint/question-type/option-count bucket is only the tooling minimum. Include multiple labels in every training bucket and do not derive risk/tool labels mechanically from model outputs.
```bash
-/path/to/laya-venv/bin/python -m tools.laya_service.evaluate /path/to/calibration-cases.jsonl --output /path/to/calibration-raw.jsonl
-/path/to/laya-venv/bin/python -m tools.laya_service.evaluate /path/to/validation-cases.jsonl --output /path/to/validation-raw.jsonl
-/path/to/laya-venv/bin/python -m tools.laya_service.calibration \
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/calibration-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/calibration-raw.jsonl
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/validation-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/validation-raw.jsonl
+dotnet run --project tools/laya_service -c Release -- calibrate \
--fit /path/to/calibration-raw.jsonl --validate /path/to/validation-raw.jsonl \
--output /path/to/calibration.json
```
-Observations retain raw pre-adapter probabilities, labels, and request hashes for detecting duplicate cases, without state. Protect these local evaluation files as potentially sensitive metadata. Calibration fits an additional scalar temperature to those probabilities, separately by checkpoint, primitive, and option count. It validates disjoint IDs, exact-request fingerprints, and provenance, reports held-out NLL, Brier score, expected calibration error, reliability bins, and risk-versus-coverage curves, and prints the artifact's SHA-256 ID. It does not retrain weights or automatically promote a provider. Inspect held-out results, especially under-routing and important language cohorts; reject a calibration that worsens acceptable risk or coverage. Temperature scaling cannot fix bad rankings or missing knowledge, and may require subsequent task-specific fine-tuning.
+The evaluator sends requests only to a literal `http://127.0.0.1:PORT/v1/decisions` endpoint, disables proxies and redirects, and writes state-free observations with raw answers, labels, provenance, and canonical request fingerprints. Treat these files as potentially sensitive metadata. Calibration fits scalar temperatures by checkpoint and `type:option-count`, validates disjoint IDs/fingerprints and homogeneous model/schema/runtime identity, and reports held-out NLL, Brier, ECE, reliability bins, and risk/coverage. The output is UTF-8 JSON with a final newline and `version: 2`, bound to model revision, schema hash, `sdk_version: "1.0.0"`, and `runtime: "NLaya"`. The CLI prints the exact artifact SHA-256.
+
+Restart with `--calibration /path/to/calibration.json` and set the printed SHA-256 in `Laya.CalibrationId`. The service rejects v1 artifacts, a different question schema, or missing checkpoint/buckets. Continue shadow evaluation and inspect held-out results before considering active mode. Temperature scaling cannot repair ranking errors or missing knowledge. No production calibration artifact ships with this repository.
-Restart the service with `--calibration /path/to/calibration.json`, put its printed SHA-256 in `Laya.CalibrationId`, and continue shadow evaluation. The service rejects a different question schema, candidate order, or a checkpoint/bucket absent from the calibration. After acceptable held-out task results, set `Laya.Mode=active` and configure confidence/margin thresholds based on that evaluation. Active mode requires a calibration ID; the response must match it. No production calibration artifact is shipped.
+## Routing Journal Report
-The existing journal report now supports both providers:
+The .NET report command handles both Jev and Laya journals. Labels use `decision_id`, `expected_tier`, and optional boolean `high_risk`. Calibration is reported separately by provider/model/rubric/checkpoint/revision/calibration/schema cohort. Unlabeled journals make no accuracy claim; reliability plots require labeled probability data.
```bash
-python3 scripts/evaluate-decision-routing.py /path/to/laya-decisions.snapshot.jsonl \
- --labels /path/to/tier-labels.jsonl --output /path/to/report.json
+dotnet run --project tools/laya_service -c Release -- report \
+ /path/to/jev-decisions.snapshot.jsonl \
+ --labels /path/to/tier-labels.jsonl --output /path/to/report.json \
+ --plot /path/to/reliability.png
```
-Tier-label rows use `decision_id`, `expected_tier`, and optional `high_risk`, as described in [Jev routing](jev-routing.md). Calibration metrics are separated by provider/model/rubric/checkpoint/revision/calibration/schema cohort. They assess raw tier distributions separately from policy safeguards and fallback. Add `--plot /path/to/reliability.png` when `matplotlib` is installed to render reliability and risk/coverage plots. Unlabeled journals make no accuracy claim. The legacy `evaluate-jev-routing.py` command remains available.
+## Rollback and Verification
-## Rollback and verification
-
-Set `Laya.Mode=disabled` and restart the gateway, or use `openclaw routing configure router --router disabled` to disable ONNX and both decision providers. Remove environment overrides that would re-enable them. Stop the independently managed Python service when it is no longer needed.
+Set `Laya.Mode=disabled` and restart the Gateway, or run `openclaw routing configure router --router disabled` to disable ONNX and both decision providers. Remove environment overrides that would re-enable them, and stop the independently managed .NET service when it is no longer needed.
```bash
-python3 -B -m unittest discover -s tests/laya-service -v
-python3 -B -m unittest discover -s tests/routing-eval -p test_jev_report.py
-dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj --filter 'FullyQualifiedName~LayaRoutingTests|FullyQualifiedName~JevRoutingTests'
+dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release
+dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -c Release \
+ -p:OpenClawSkipDashboardBuild=true --filter FullyQualifiedName~LayaRoutingTests
```
-The Python unit tests require only the standard library; they do not download weights or make hosted inference calls. Real local inference is a separate operator smoke check. This implementation covers turn routing, local serving, compatibility improvements, and calibration tooling. Skill selection, memory reranking, and workflow escalation remain later integrations requiring their own quality evaluations.
+These tests use fakes and fixtures; they do not download weights or exercise real model quality. Real inference and checkpoint parity are opt-in operator checks in an environment with model assets and the required device backend. Skill selection, memory reranking, and workflow escalation remain separate integrations requiring their own evaluations.
+
+For an opt-in real-model check, download all three checkpoints, start the service once per checkpoint by setting `--checkpoint english`, `--checkpoint multilingual`, or `--checkpoint typed-decisions`, then evaluate the same labeled parity dataset against each service:
+
+```bash
+dotnet run --project tools/laya_service -c Release -- serve \
+ --manifest /path/to/laya-models/manifest.json --checkpoint english --device cpu
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/parity-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/english-observations.jsonl
+```
-Development verification on September 21, 2026 included real English inference on Apple MPS, multilingual inference on CPU (including Armenian checkpoint selection), a compiled macOS arm64 NativeAOT client calling the local service, and a synthetic calibration/evaluation round trip. These verify the integration, not production quality or a general latency guarantee. No production routing mode or model weights were changed by that verification.
+Repeat with `multilingual` and `typed-decisions` (and distinct output paths). Compare each observation to an approved reference for exact typed-answer/category matches and probability deltas within a tolerance chosen before the run for the same device/runtime. Record checkpoint, revision, runtime, device, and tolerance with the results. The repository does not supply real-model golden outputs or a production acceptance threshold; this check is not part of ordinary CI and is not evidence of production routing quality.
diff --git a/docs/superpowers/plans/2026-09-27-nlaya-laya-service.md b/docs/superpowers/plans/2026-09-27-nlaya-laya-service.md
new file mode 100644
index 00000000..daccf5b8
--- /dev/null
+++ b/docs/superpowers/plans/2026-09-27-nlaya-laya-service.md
@@ -0,0 +1,323 @@
+# NLaya 本地 Laya 服务迁移实施计划
+
+> **面向 agentic workers：** 实施时必须逐任务执行本计划，并在每个任务后独立验证。可选择 `superpowers:subagent-driven-development`（推荐）或 `superpowers:executing-plans`。步骤使用 checkbox (`- [ ]`) 跟踪。
+
+**目标：** 将 `tools/laya_service` 与 Laya/Jev 路由评估工作流迁为不依赖 Python 的 .NET 10 工具，同时保持 Gateway 当前 loopback HTTP 契约和安全行为。
+
+**架构：** 在 `tools/laya_service` 建立独立 .NET 10 CLI 与测试项目。服务通过 `NLaya`/`NLaya.TorchSharp` 使用本地模型资产，不把推理依赖引入 Gateway；共享 wire protocol、manifest、校准产物和报告逻辑均在 .NET 实现。先迁移并验证新实现，再删除 Python 模块、入口和测试。
+
+**技术栈：** .NET 10、NLaya 1.0.0、NLaya.TorchSharp 1.0.0、TorchSharp-cpu 0.107.0、xUnit v3、ASP.NET Core shared framework、ScottPlot 5.1.59、GitHub Actions。
+
+## 全局约束
+
+- `tools/laya_service` 使用 .NET 10，并固定引用 NuGet 包 `NLaya` 1.0.0。
+- 必须包含 TorchSharp CPU runtime；只有在受支持且显式选择的目标平台中才包含加速器 runtime 包。
+- Gateway 不直接加载 NLaya 或 TorchSharp；维持 OpenClaw Gateway 当前使用的本地 HTTP 边界。
+- 保留 `GET /health`、`POST /v1/decisions`、模型标识 `laya@`、现有安全策略以及默认禁用路由行为。
+- 这些工作流不得依赖 Python、pip 或 Hugging Face Python CLI；不要求整个仓库去 Python 化。
+- 模型下载固定到 40 位 Hugging Face commit 并校验逐文件 SHA-256；普通 CI 不下载模型权重。
+- 校准产物使用格式版本 2；不得在新 runtime 下静默接受现有 Python 校准产物。
+- 不得根据 parity 或合成测试宣称路由质量已达到生产要求。
+- 未在受支持 TorchSharp 目标上成功发布并运行前，不得宣称服务支持 NativeAOT。
+
+## 文件结构
+
+- Create `tools/laya_service/LayaService.csproj`：服务可执行项目、NuGet 依赖及 rubric 的嵌入资源。
+- Create `tools/laya_service/Program.cs`、`CommandLine.cs`：命令派发、参数解析、退出码和顶层安全错误处理。
+- Create `tools/laya_service/Protocol/WireModels.cs`、`StrictJson.cs`、`RequestValidator.cs`：HTTP wire DTO、重复 JSON key 拒绝和请求合同校验。
+- Create `tools/laya_service/Hosting/DecisionServer.cs`：loopback HTTP 服务、安全响应、限流和 inference gate。
+- Create `tools/laya_service/Hosting/ServiceContracts.cs`、`Inference/ServeOptions.cs`、`Evaluation/Observation.cs`：跨 host、runtime 和 evaluation 共用的 options/observation contract。
+- Create `tools/laya_service/Models/ModelManifest.cs`、`HuggingFaceDownloader.cs`：模型 manifest、资产白名单、下载、hash 与原子文件操作。
+- Create `tools/laya_service/Inference/NLayaDecisionPredictor.cs`、`CalibrationStore.cs`：NLaya checkpoint/router 生命周期、typed-answer 适配、token budget 校验和校准应用。
+- Create `tools/laya_service/Evaluation/CaseEvaluator.cs`、`CalibrationFitter.cs`、`CalibrationMetrics.cs`：JSONL 样本推理、温度拟合与指标。
+- Create `tools/laya_service/Reporting/RoutingJournalReport.cs`、`ReliabilityPlot.cs`：Jev/Laya 决策日志报告与可选 PNG 图。
+- Create `tools/laya_service/tests/LayaService.Tests.csproj` 及按协议、host、model、evaluation、report 分组的 xUnit 测试。
+- Modify `OpenClaw.Net.slnx`：加入工具和测试项目。
+- Modify `src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs`、`src/OpenClaw.Tests/LayaRoutingTests.cs`：验证新 runtime metadata，同时保持 Gateway fallback。
+- Modify `src/OpenClaw.Routing.Decisions/OpenClaw.Routing.Decisions.csproj` only if the rubric is moved; preferred implementation keeps `tools/laya_service/rubrics/openclaw-laya-tiers-v1.json` at its current path, so no project-file change is needed.
+- Modify `.github/workflows/ci.yml`：以 .NET 工具测试替换 Laya 与 routing-report Python 测试步骤。
+- Modify `docs/laya-routing.md`、`docs/jev-routing.md`、`docs/cli/routing.md`、`tools/laya_service/README.md`、`docs/README.md`、`docs/SITE_MAP.md`、`docs/zh-CN/SITE_MAP.md`；create `docs/zh-CN/integrations/laya-routing.md`。
+- Delete after .NET coverage passes: `tools/laya_service/__init__.py`, `__main__.py`, `calibration.py`, `compat.py`, `download.py`, `evaluate.py`, `protocol.py`, `runtime.py`, `requirements.txt`, `tests/laya-service/test_service.py`, `tests/routing-eval/test_jev_report.py`, `scripts/evaluate-decision-routing.py`, `scripts/evaluate-jev-routing.py`.
+- Keep `tools/laya_service/rubrics/**`, `tools/laya_service/licenses/**`, `tools/laya_service/THIRD_PARTY_NOTICES.md`, and routing JSONL sample fixtures; these remain inputs/resources, not Python code.
+
+---
+
+### Task 1：建立 .NET CLI 与测试骨架
+
+**文件：**
+
+- Create: `tools/laya_service/LayaService.csproj`
+- Create: `tools/laya_service/Program.cs`
+- Create: `tools/laya_service/CommandLine.cs`
+- Create: `tools/laya_service/tests/LayaService.Tests.csproj`
+- Create: `tools/laya_service/tests/CommandLineTests.cs`
+- Modify: `OpenClaw.Net.slnx`
+
+**接口：**
+
+- `CommandLine.Parse(string[] args)` 产出 `CommandInvocation`，其 `Command` 为 `serve|download|evaluate|calibrate|report`，`Options` 保留重复 `--checkpoint` 值并拒绝未知选项。
+- `Program` 只解析命令、调用相应 handler、将安全错误写到 stderr 并返回非零退出码；不得输出请求 state、凭据或任意底层异常文本。
+
+- [x] **步骤 1：先写 CLI 解析失败测试**，覆盖五个命令、重复 checkpoint、缺值参数、未知命令和未知参数。例如：
+
+```csharp
+[Fact]
+public void Parse_DownloadPreservesRepeatedCheckpoints()
+{
+ var parsed = CommandLine.Parse(["download", "--destination", "models", "--checkpoint", "english", "--checkpoint", "multilingual"]);
+ Assert.Equal("download", parsed.Command);
+ Assert.Equal(new[] { "english", "multilingual" }, parsed.Options.GetMany("checkpoint"));
+}
+```
+
+- [x] **步骤 2：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter FullyQualifiedName~CommandLineTests`
+预期：因项目或 `CommandLine` 尚不存在而失败。
+
+- [x] **步骤 3：创建项目骨架。** `LayaService.csproj` 设置 `OutputType=Exe`、`TargetFramework=net10.0`、root namespace；引用 `NLaya` 1.0.0、`NLaya.TorchSharp` 1.0.0、`TorchSharp-cpu` 0.107.0、`Microsoft.AspNetCore.App` 和 ScottPlot 5.1.59。测试项目采用 `eng/nacos-live/tests/NacosLiveAcceptance.Tests.csproj` 的 xUnit v3 模式，并引用服务项目。把两个项目放入 `.slnx` 的 `/tools/laya_service/` solution folder。
+- [x] **步骤 4：实现命令解析和顶层派发。** 每个命令 handler 初期可以返回明确的 `not_implemented` 错误；不得留下成功但无动作的空实现。保持 CLI 参数为 `dotnet run --project tools/laya_service -- ...`。
+- [x] **步骤 5：重跑 CLI 测试并构建项目。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release`
+预期：CLI 解析测试通过。
+运行：`dotnet build tools/laya_service/LayaService.csproj -c Release`
+预期：构建成功，warnings-as-errors 下无警告。
+
+- [ ] **步骤 6：提交** `feat: scaffold NLaya local service CLI`。
+
+### Task 2：实现严格 wire protocol 和安全 loopback server
+
+**文件：**
+
+- Create: `tools/laya_service/Protocol/WireModels.cs`
+- Create: `tools/laya_service/Protocol/StrictJson.cs`
+- Create: `tools/laya_service/Protocol/RequestValidator.cs`
+- Create: `tools/laya_service/Hosting/DecisionServer.cs`
+- Create: `tools/laya_service/tests/ProtocolTests.cs`
+- Create: `tools/laya_service/tests/DecisionServerTests.cs`
+
+**接口：**
+
+- 在 `Hosting/ServiceContracts.cs` 定义 `IDecisionPredictor.PredictAsync(DecisionWireRequest request, CancellationToken cancellationToken)`、`GetHealth()` 和 `ServiceOptions(int Port, int MaxConnections, int MaxRequestBytes, TimeSpan RequestTimeout)`；健康方法返回 health JSON DTO。测试使用 fake predictor。
+- `RequestValidator.Validate(DecisionWireRequest request, string configuredModel)` 拒绝无效请求并返回固定 reason code。
+- `DecisionServer.Build(IDecisionPredictor predictor, ServiceOptions options)` 返回可由测试启动的 `WebApplication`；生产只监听 `IPAddress.Loopback`。
+
+- [x] **步骤 1：先写请求与 host 测试。** 覆盖 golden request、非法字段/type/大小、duplicate JSON key、Origin、Host、chunked/超大 body、错误码、并发 inference gate、响应不回显输入。至少固定 HTTP 状态与错误响应：无效输入 `422`；busy 和 predictor 异常 `503`。
+- [x] **步骤 2：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~ProtocolTests|FullyQualifiedName~DecisionServerTests'`
+预期：测试类型尚未实现时失败。
+
+- [x] **步骤 3：实现 DTO 和 JSON 校验。** wire request 仅允许 `model/state/questions/rubric_version/language`；保留 choice/score/noul 形状、候选项顺序和已确认的请求界限。用 `Utf8JsonReader`/`JsonDocument` 严格拒绝重复 key、非有限数值和无效 UTF-8；不可将 JSON 对象重序列化后再计算 schema identity。
+- [x] **步骤 4：实现 ASP.NET Core server。** 绑定 loopback；只映射 `GET /health` 和 `POST /v1/decisions`；校验 Host 精确匹配绑定地址/端口并拒绝任意 Origin；要求 JSON `Content-Length`、拒绝 Transfer-Encoding、限制 body 为 65,536 bytes；无排队单推理 gate、有限连接数、连接/读取 deadline；关闭请求日志。以固定 reason code 返回 4xx/5xx，不回显异常。
+- [x] **步骤 5：重跑服务测试。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~ProtocolTests|FullyQualifiedName~DecisionServerTests'`
+预期：fake predictor 下所有 wire、安全和限制测试通过，不需要模型文件。
+
+- [ ] **步骤 6：提交** `feat: add secure local Laya decision endpoint`。
+
+### Task 3：实现 manifest、模型下载和资产完整性检查
+
+**文件：**
+
+- Create: `tools/laya_service/Models/ModelManifest.cs`
+- Create: `tools/laya_service/Models/HuggingFaceDownloader.cs`
+- Create: `tools/laya_service/tests/ModelManifestTests.cs`
+- Create: `tools/laya_service/tests/HuggingFaceDownloaderTests.cs`
+- Modify: `tools/laya_service/CommandLine.cs`
+- Modify: `tools/laya_service/Program.cs`
+
+**接口：**
+
+- `ModelManifest.LoadAndVerify(string manifestPath, string expectedRevision)` 检查 manifest `version=1`、revision、checkpoint allowlist、路径 containment 和每个文件的 SHA-256，并返回 `VerifiedManifest`（checkpoint 名称、revision、绝对路径及文件哈希）。校准产物单独使用 `version=2`，二者不可混淆。
+- `HuggingFaceDownloader.DownloadAsync(DownloadOptions options, HttpClient http, CancellationToken ct)` 仅从 `https://huggingface.co/convaiinnovations/laya/resolve/{revision}/{file}` 获取 allowlist 文件，输出模型 manifest 路径。
+- 默认 revision 沿用现有实现的 `1c5edc17a7acd8701df6fc341c0d179f1c62c982`；迁移 `download.py` 中该 revision 的已知逐文件哈希。只有文件下载后哈希验证通过，才将其写入 manifest。
+
+- [x] **步骤 1：先写 manifest 与下载测试。** 覆盖固定 revision、三种 checkpoint 路径、缺失/改坏文件、路径穿越、未知文件、HTTP 错误/重定向、哈希不匹配、原子 manifest 以及许可/notice 输出；使用假的 `HttpMessageHandler`，不访问网络。
+- [x] **步骤 2：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~ModelManifestTests|FullyQualifiedName~HuggingFaceDownloaderTests'`
+预期：实现缺失导致失败。
+
+- [x] **步骤 3：实现下载与校验。** 文件清单从已固定的 NLaya checkpoint 文件布局生成；每个下载写入 destination 下唯一临时文件，摘要通过后才原子替换；不同 revision 使用不同目标目录；验证任何最终路径都留在 destination 内。写入 manifest 前重新验证完整 checkpoint。复制现有 `THIRD_PARTY_NOTICES.md` 和 Apache-2.0 license 到模型目录。
+- [x] **步骤 4：连接 `download` CLI。** 支持 `--destination`、`--revision` 和可重复 `--checkpoint`；缺少必需值或 revision 不为 40 位 hex 时返回非零退出码。
+- [x] **步骤 5：重跑下载测试与 Release build。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~ModelManifestTests|FullyQualifiedName~HuggingFaceDownloaderTests'`
+预期：所有测试通过且测试未产生实际 Hugging Face 请求。
+
+- [ ] **步骤 6：提交** `feat: add pinned NLaya model downloader`。
+
+### Task 4：接入 NLaya runtime、checkpoint routing 和新 metadata
+
+**文件：**
+
+- Create: `tools/laya_service/Inference/NLayaDecisionPredictor.cs`
+- Create: `tools/laya_service/Inference/CalibrationStore.cs`
+- Create: `tools/laya_service/tests/NLayaDecisionPredictorTests.cs`
+- Modify: `tools/laya_service/Program.cs`
+- Modify: `src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs`
+- Modify: `src/OpenClaw.Tests/LayaRoutingTests.cs`
+
+**接口：**
+
+- 在 `Inference/ServeOptions.cs` 定义 `ServeOptions(string ManifestPath, string? CalibrationPath, int Port, string Device, string Checkpoint, int Threads)`。`NLayaDecisionPredictor.LoadAsync(VerifiedManifest manifest, ServeOptions options, CancellationToken ct)` 先校验所有模型与 calibration，再加载/warm-up 选中 checkpoint，返回 ready predictor。
+- Predictor 映射 `DecisionWireRequest` 为 NLaya `Questions`/state/`RouteOptions`，再把 typed result 映射到现有 `DecisionWireResponse`。
+- Metadata 必须提供现有字段并固定 `sdk_version="1.0.0"`、`runtime="NLaya"`、`revision`、`checkpoint`、`schema_hash`、`rubric_version`、`device`、`calibration_id` 和 `truncated=false`。
+
+- [x] **步骤 1：先写 fake/fixture 输出适配测试。** 测试 Choice/Score/Noul 与概率映射、raw_answers 保留、自动 English/multilingual 路由、语言 hint、非拉丁/混合文本、显式 typed-decisions、未知 checkpoint、设备不可用和输入超过 token budget 时拒绝。typed-decisions 不可因任务名称自动选择。
+- [x] **步骤 2：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter FullyQualifiedName~NLayaDecisionPredictorTests`
+预期：NLaya adapter 尚未实现时失败。
+
+- [x] **步骤 3：实现 NLaya adapter。** 从通过 hash 校验的本地路径加载；使用 `NLaya.Routing.Router`/NLaya 1.0.0 公开 API 和 `UseTorchSharp`；禁用 task-name 自动选择；仅预加载 manifest 中请求的 checkpoints。使用同一 tokenizer 逻辑检查 token budget，超限在 inference 前失败，不将 `truncated=false` 用作未经检查的声明。明确配置 CPU；accelerator 仅在对应 runtime 被安装且显式选择时初始化，不捕获后退到 CPU。
+- [x] **步骤 4：更新 Gateway metadata contract。** 把 `LayaDecisionClient` 的 SDK 检查改为 `1.0.0` 并新增严格 `runtime == "NLaya"` 检查。扩展 `LayaRoutingTests` 的 handler metadata 和 mismatched runtime 用例，保持服务错误时 baseline fallback 与无 hosted fallback 断言。
+- [x] **步骤 5：运行服务 adapter 与 Gateway 定向测试。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter FullyQualifiedName~NLayaDecisionPredictorTests`
+预期：fake/fixture 测试通过，无权重下载。
+运行：`dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -c Release -p:OpenClawSkipDashboardBuild=true --filter FullyQualifiedName~LayaRoutingTests`
+预期：Gateway metadata 和原有 fallback 测试通过。
+
+- [ ] **步骤 6：提交** `feat: adapt NLaya predictions to Laya wire contract`。
+
+### Task 5：迁移样本 evaluation 与 v2 calibration
+
+**文件：**
+
+- Create: `tools/laya_service/Evaluation/CaseEvaluator.cs`
+- Create: `tools/laya_service/Evaluation/CalibrationFitter.cs`
+- Create: `tools/laya_service/Evaluation/CalibrationMetrics.cs`
+- Create: `tools/laya_service/tests/CaseEvaluatorTests.cs`
+- Create: `tools/laya_service/tests/CalibrationFitterTests.cs`
+- Modify: `tools/laya_service/Inference/CalibrationStore.cs`
+- Modify: `tools/laya_service/CommandLine.cs`
+- Modify: `tools/laya_service/Program.cs`
+
+**接口：**
+
+- `CaseEvaluator.EvaluateAsync(string datasetPath, Uri endpoint, string outputPath, HttpClient http, CancellationToken ct)` 输出 JSONL observation 数量。
+- 在 `Evaluation/Observation.cs` 定义 `Observation(string CaseId, string CaseFingerprint, string QuestionId, string Model, string Checkpoint, string SchemaHash, string RuntimeVersion, JsonElement RawAnswer, string Label)`；`RawAnswer` 保留答案类型及原始概率，不含 state。
+- `CalibrationFitter.Fit(IReadOnlyList training, IReadOnlyList validation, int minimumSamples = 20)` 返回 v2 calibration DTO。
+- `CalibrationMetrics.Measure(IReadOnlyList rows)` 返回样本数、accuracy、NLL、Brier、ECE、reliability bins 和 risk/coverage；`LabeledPrediction` 在 `CalibrationMetrics.cs` 定义，携带答案类型、候选键、归一化概率和标签。
+
+- [x] **步骤 1：先写 evaluation 隐私与 HTTP 测试。** 用 fake `HttpMessageHandler` 验证 case ID 唯一性、标签覆盖每个问题、默认 rubric/model、loopback literal endpoint 限制、proxy/redirect 禁用、服务 metadata identity 检查、canonical 请求指纹及 observation 不含原始 state。
+- [x] **步骤 2：先写 calibration metric/contract 测试。** 覆盖概率归一化、noul/choice/score 分布、temperature grid、train/validation ID 与 fingerprint 重叠、模型/schema/runtime 混合、bucket 不一致、每桶最少 20 条、多标签、稳定 v2 JSON 和 exact-byte SHA-256。保留旧 Python metrics golden 结果。
+- [x] **步骤 3：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~CaseEvaluatorTests|FullyQualifiedName~CalibrationFitterTests'`
+预期：尚无实现时失败。
+
+- [x] **步骤 4：实现 evaluator。** 读取现有 case JSONL，每条请求只发往字面 loopback `/v1/decisions`；用 `HttpClientHandler` 设置 `AllowAutoRedirect=false` 且禁用代理。observations 只写 ID、canonical fingerprint、question、label、模型/checkpoint/schema/runtime provenance 和 raw answer；写入临时文件，成功处理整个数据集后原子替换目标。
+- [x] **步骤 5：实现 metrics、温度拟合和 v2 calibration store。** 拟合继续使用候选温度 `10 ** (-1 + i / 80)`，`i=0..160` 的 NLL 最小值；按 checkpoint 与 `type:option-count` 分桶。artifact 固定为 UTF-8 JSON、final newline、`version=2`，绑定模型 revision、schema hash、NLaya 版本、温度和 validation metrics；service 校验 configured SHA-256，拒绝 v1 并提示重新生成。
+- [x] **步骤 6：连接 `evaluate` 与 `calibrate` CLI 并重跑定向测试。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~CaseEvaluatorTests|FullyQualifiedName~CalibrationFitterTests'`
+预期：原始观察隐私、校准分桶及 artifact 哈希测试全部通过。
+
+- [ ] **步骤 7：提交** `feat: port Laya evaluation and calibration to dotnet`。
+
+### Task 6：迁移 provider-neutral routing journal report
+
+**文件：**
+
+- Create: `tools/laya_service/Reporting/RoutingJournalReport.cs`
+- Create: `tools/laya_service/Reporting/ReliabilityPlot.cs`
+- Create: `tools/laya_service/tests/RoutingJournalReportTests.cs`
+- Create: `tools/laya_service/tests/ReliabilityPlotTests.cs`
+- Keep as fixtures: `tests/routing-eval/sample-routing-baseline.json`
+- Keep as fixtures: `tests/routing-eval/turn-routing-quality.sample.jsonl`
+- Modify: `tools/laya_service/CommandLine.cs`
+- Modify: `tools/laya_service/Program.cs`
+
+**接口：**
+
+- `RoutingJournalReport.ReadJsonLines(string path)` 对每行给出带行号的 JSON 错误；`Summarize(rows, labels)` 返回与当前脚本字段兼容的 report DTO。
+- `ReliabilityPlot.WritePng(RoutingReport report, string outputPath)` 使用 ScottPlot 5.1.59 生成 reliability 与 risk/coverage 两列图；没有带标签概率 cohort 时返回明确错误。
+
+- [x] **步骤 1：先写 report parity tests。** 从现有 `test_jev_report.py` 逐例移植 Jev fallback、Laya cohort 分组、abstain 概率、无标签不报 accuracy、空/重复/unmatched 数据校验；断言输出字段、数值和 JSON key 与现有 Python fixtures 一致。
+- [x] **步骤 2：运行测试确认失败。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~RoutingJournalReportTests|FullyQualifiedName~ReliabilityPlotTests'`
+预期：尚无报告实现时失败。
+
+- [x] **步骤 3：实现 summary/quality/calibration cohort。** 保留 decisions、usage、eligible proposals、coverage、mode/provider/model/rubric/reason/tier counts、latency percentile、token/cost、quality confusion/F1/under/over-routing/high-risk retention 字段；概率指标调用 Task 5 的 `CalibrationMetrics` 并严格按 provider/model/rubric/checkpoint/revision/calibration/schema 分组。
+- [x] **步骤 4：实现 ScottPlot 输出。** 锁定 ``；生成 headless PNG，标题包含 cohort identity，绘出理想 calibration 对角线和 risk/coverage 曲线；无图像设备依赖，不生成无数据的误导图。
+- [x] **步骤 5：连接 `report` CLI 并运行 fixtures/PNG 测试。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --filter 'FullyQualifiedName~RoutingJournalReportTests|FullyQualifiedName~ReliabilityPlotTests'`
+预期：Jev 与 Laya fixtures 的 report JSON parity 通过，PNG 文件存在且非空。
+
+- [ ] **步骤 6：提交** `feat: port decision routing reports to dotnet`。
+
+### Task 7：接入 solution/CI 并移除 Python 实现
+
+**文件：**
+
+- Modify: `OpenClaw.Net.slnx`
+- Modify: `.github/workflows/ci.yml`
+- Delete: Task 1–6 文件清单中的 Python modules、Python requirements、Python 测试与两个 report entrypoint。
+- Keep: rubric、license、notice 和 routing JSONL fixtures。
+
+- [x] **步骤 1：先确认 .NET 测试通过，再更新 CI 命令。** 在 `ci.yml` 的本地决策工具步骤中使用 `dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release --no-build`；保留仓库其他既有 Python 检查，不删除 `eng/verify-vault-publish-boundary.py` 步骤。
+- [x] **步骤 2：将服务和测试项目加入 `.slnx` 并做 solution restore/build。**
+
+运行：`dotnet restore OpenClaw.Net.slnx`
+预期：NLaya/TorchSharp/ScottPlot 固定依赖可恢复。
+运行：`dotnet build OpenClaw.Net.slnx --no-restore -c Release`
+预期：solution build 成功。
+
+- [x] **步骤 3：仅在新测试已通过后删除 Python 实现/测试/入口。** 保留许可、rubric 和 JSONL fixtures。用 `rg -n "tools\.laya_service|tools/laya_service|evaluate-jev-routing|evaluate-decision-routing|tests/laya-service|test_jev_report"` 扫描 tracked 文件；仅允许新文档里说明历史迁移的文字，不允许剩余运行时引用。
+- [x] **步骤 4：运行 CI 对应的 Python 依赖边界检查与 .NET tests。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release`
+预期：所有工具测试通过。
+运行：`dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -c Release -p:OpenClawSkipDashboardBuild=true --filter FullyQualifiedName~LayaRoutingTests`
+预期：Gateway contract 与安全 fallback 测试通过。
+运行：`dotnet build OpenClaw.Net.slnx -c Release`
+预期：完整 solution build 通过。
+
+- [ ] **步骤 5：提交** `build: replace Python Laya tooling with dotnet`。
+
+### Task 8：迁移中英文文档并执行最终验收
+
+**文件：**
+
+- Modify: `docs/laya-routing.md`
+- Modify: `docs/jev-routing.md`
+- Modify: `docs/cli/routing.md`
+- Modify: `tools/laya_service/README.md`
+- Modify: `docs/README.md`
+- Modify: `docs/SITE_MAP.md`
+- Modify: `docs/zh-CN/SITE_MAP.md`
+- Create: `docs/zh-CN/integrations/laya-routing.md`
+- Modify: `docs/cli/routing.md` only once; consolidate references into the same edit.
+
+- [x] **步骤 1：将运维命令改为 .NET CLI。** 中英文文档说明 NuGet/runtime 版本、`serve/download/evaluate/calibrate/report` 命令、revision/hash/manifest、runtime metadata、校准 v2 重生成、设备选择和 rollback；Jev 报告改为同一 `report` 命令，旧 Python 命令不再展示。
+- [x] **步骤 2：更新文档导航。** 在 `docs/README.md`、`docs/SITE_MAP.md` 和 `docs/zh-CN/SITE_MAP.md` 链接运维指南及中文版；确保新增相对链接目标存在。
+- [x] **步骤 3：运行最终 focused gates。**
+
+运行：`dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release`
+预期：全部 service/download/evaluate/calibrate/report tests 通过，且不访问 Hugging Face。
+运行：`dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -c Release -p:OpenClawSkipDashboardBuild=true --filter FullyQualifiedName~LayaRoutingTests`
+预期：Gateway 新 metadata 与 baseline fallback tests 通过。
+运行：`dotnet build OpenClaw.Net.slnx -c Release`
+预期：所有 solution projects 编译通过。
+运行：`git diff --check`
+预期：无空白错误。
+
+- [x] **步骤 4：提供 opt-in 真实模型验证说明。** 记录需由操作者在模型下载和硬件已配置的环境运行的命令：分别 smoke 每个已支持 checkpoint，检查 parity 的 typed-answer/category 与概率容差；不得把此项混入普通 CI，也不得称为生产质量证据。
+- [ ] **步骤 5：提交** `docs: document NLaya Laya service migration`。
+
+## 规格覆盖自检
+
+- 独立 CLI、精确 package/runtime、Gateway 进程边界：Tasks 1、4、7。
+- 模型下载、revision/hash/manifest、许可归属与离线启动：Tasks 3、4。
+- Loopback HTTP、输入/token 限制、错误码、并发与隐私：Task 2、4。
+- Python 去除及所有原运行路径无 Python 依赖：Tasks 5–8。
+- evaluation、校准、v2 artifact、指标与 privacy：Task 5。
+- Jev/Laya provider-neutral report、PNG 与旧入口移除：Tasks 6–8。
+- Gateway metadata/fallback contract、CI、文档和可选真实模型 smoke：Tasks 4、7、8。
diff --git a/docs/superpowers/specs/2026-09-27-nlaya-laya-service-design.md b/docs/superpowers/specs/2026-09-27-nlaya-laya-service-design.md
new file mode 100644
index 00000000..85ae085a
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-27-nlaya-laya-service-design.md
@@ -0,0 +1,77 @@
+# NLaya 本地 Laya 服务迁移设计
+
+## 状态
+
+设计内容已在对话中确认；实现规划前，等待用户审阅本文档。
+
+## 目标
+
+将 `tools/laya_service` 下的 Python 实现和工具替换为独立的 .NET 10 CLI，使用固定版本的 NuGet 包 `NLaya` 1.0.0。该工具负责本地推理、checkpoint 下载与校验、评估、校准以及 provider-neutral 路由报告。这些工作流不得依赖 Python、pip 或 Hugging Face Python CLI。
+
+保留 OpenClaw Gateway 当前使用的本地 HTTP 边界。Gateway 不直接加载 NLaya 或 TorchSharp。保留现有模型 revision 标识、安全策略以及默认禁用路由的行为。不得根据 parity 或合成测试宣称路由质量已达到生产要求。
+
+## 非目标
+
+- 不把 NLaya 集成进 Gateway，也不更改 Jev/ONNX 提供方选择。
+- 不修改模型权重、不自动执行校准，也不默认启用 active Laya 路由。
+- 不移除无关 Python 工具，也不要求整个仓库去 Python 化。
+- 未成功完成发布和运行验证前，不宣称 TorchSharp 服务支持 NativeAOT。
+- 不在新 runtime 下静默接受现有 Python 校准产物。
+
+## 组件与依赖
+
+在原路径 `tools/laya_service` 建立 .NET 10 可执行项目，替换该目录中的 Python 包。引用 `NLaya` 1.0.0 和匹配版本的 `NLaya.TorchSharp` 包。必须包含 TorchSharp CPU runtime；只有在受支持且显式选择的目标平台中才包含加速器 runtime 包。独立服务拥有这些依赖；Gateway 继续使用现有 HTTP client。
+
+提供 `serve`、`download`、`evaluate`、`calibrate` 和 `report` CLI 命令。在可行时保留现有选项名称和含义：
+
+- `serve`：manifest、校准文件、端口、设备、checkpoint 和线程数限制。
+- `download`：目标目录、不可变模型 revision 和一个或多个 checkpoint。
+- `evaluate`：case JSONL、本机 loopback endpoint 和 observations 输出路径。
+- `calibrate`：互不重叠的原始拟合与验证 observations，以及输出产物路径。
+- `report`：路由日志、可选标签、JSON 输出，以及可选 reliability/risk-coverage 图表。
+
+Python 模块调用方式将由 .NET 可执行文件或 `dotnet run --project tools/laya_service -- ...` 取代。相应更新所有运维示例。
+
+## 下载与推理数据流
+
+`download` 命令使用 .NET HTTP API，从 Hugging Face 仓库 `convaiinnovations/laya` 获取 NLaya 所需文件，并固定到 40 位 commit。不得调用 `hf` 或 Python，也不得使用未固定的分支。使用前校验每个文件的 SHA-256；下载内容先写入临时文件；manifest 以原子方式写入，记录 manifest 版本、revision、checkpoint 路径、文件哈希和 NLaya 版本。上游模型许可和归属说明与下载资产放在一起。不同 revision 使用不同目标目录。
+
+保留当前模型 revision 作为初始候选。切换门槛之一是针对每个受支持 checkpoint 完成真实模型 smoke test；若该 revision 与 NLaya 不兼容，只有同时更新固定版本和哈希，并取得通过的 parity 证据后才能更改。CI 使用模拟 HTTP 测试，不下载模型权重。
+
+启动时，`serve` 校验 manifest 和全部资产哈希，验证 runtime/package 与校准身份，加载并 warm-up 配置的 checkpoint，之后才绑定 `127.0.0.1`。推理过程离线运行，只加载本地模型路径。自动路由可选择 English 和 multilingual checkpoint；`typed-decisions` 必须显式指定，绝不能根据任务名称自动选择。设备必须显式选择；若对应 runtime 或硬件不可用则失败，不得静默退回 CPU。
+
+保留 `GET /health`、`POST /v1/decisions`、请求字段、模型标识 `laya@`、typed-answer 响应结构及现有 loopback 保护。服务将 NLaya 结果适配为现有 wire response。保留 metadata 字段名，将 `sdk_version` 设为 `1.0.0`，并新增 `runtime: "NLaya"`；更新 `LayaDecisionClient`，使其与现有 revision、checkpoint、schema、rubric、device、truncation 和 calibration 校验一起验证这两个值。对于超过已验证 tokenizer token 预算的请求，服务必须拒绝而不是截断；只有完成检查后才能报告 `truncated: false`。
+
+使用 golden cases 验证问题顺序、规范化 schema hash、checkpoint 选择（包括非拉丁文字和混合文字脚本）、答案映射及概率容差。未知或不支持的输入一律 fail closed。
+
+## 评估与校准
+
+.NET `evaluate` 命令读取现有带标签 case JSONL 格式，校验 case ID 唯一性和标签，并且只向字面值为 loopback 地址的 `/v1/decisions` endpoint 发送请求，不使用代理或跟随重定向。输出 raw observations，包含 case ID、规范化请求指纹、问题 ID、模型、checkpoint、schema hash、runtime 版本、原始答案和标签。observations 中绝不写入 state 或 prompt 文本。
+
+.NET `calibrate` 命令分别读取拟合和 held-out observation 文件。拒绝重复或重叠的 ID、ID 不同但请求指纹相同的数据、混合的 model/schema/runtime 身份、缺失的 checkpoint/type/option-count 分桶、样本不足以及缺少多个标签的分桶。保留现有概率温度缩放行为和最小样本策略。报告 raw 与 calibrated 预测的 accuracy、NLL、Brier score、ECE、reliability bins 和 risk/coverage。校准只调整概率，不重新训练权重，也不启用提供方。
+
+校准产物使用格式版本 2。每个产物绑定模型 revision、question schema、NLaya 版本、按 checkpoint 分桶的温度以及 held-out 指标。产物序列化为确定性的 UTF-8 JSON，并以换行符结尾；calibration ID 是对产物精确字节计算的 SHA-256。服务只接受匹配的 v2 产物；active routing 仍要求配置该产物的 SHA-256。对 v1 Python 生成的校准产物明确拒绝，并提示重新生成；不随代码发布生产校准产物。
+
+将 provider-neutral 决策日志报告迁移到 .NET CLI。为 Jev 和 Laya 保留现有 JSON 字段及报告行为：摘要计数、延迟/成本、提案覆盖率/差异、可选的人工 tier 标签质量，以及按 provider/model/rubric/checkpoint/revision/calibration/schema 分组的 raw probability calibration 指标。使用固定版本的 .NET 绘图库保留可选 reliability 和 risk/coverage 图。移除导入 `tools.laya_service.calibration` 的 Python 报告入口，并同步更新文档和测试，改用 .NET 命令。
+
+## 错误处理与安全
+
+manifest、资产摘要、package/runtime 身份、checkpoint 或 calibration 无效时，服务须在开放 socket 前启动失败。服务仅绑定 loopback；拒绝非预期 Host 和任何 Origin；拒绝重定向、chunked 或超大请求体；限制连接数和并发推理；不记录请求或请求体日志。无效请求返回 422 和固定 reason code；过载与推理失败返回 503 和固定的非敏感错误。绝不回显 state、header、凭据或依赖异常原文。
+
+下载时验证路径始终位于目标目录内，并原子替换资产/manifest。Runtime 不下载模型。校准和评估输出不含 state 或凭据。服务错误时保留 Gateway 现有 baseline fallback 行为，不新增 hosted fallback。
+
+## 测试与验证
+
+- 将现有 protocol、manifest、calibration 和 report 测试迁移为聚焦的 .NET 测试；保留 canonical hash 与报告结构的 golden assertions。
+- 使用 fake predictor 测试服务 HTTP 安全和限制，包括无效 JSON、重复 key、禁止的 Host/Origin、超大/chunked 请求体、过载和不回显错误。
+- 使用模拟 HTTP 测试 downloader allowlist、固定 revision、摘要不匹配、路径穿越、manifest 原子写入以及许可/归属信息输出。
+- 测试 evaluation 隐私/溯源，以及 calibration 数据集划分、分桶、最小样本数、格式版本、稳定产物哈希和 held-out 指标行为。
+- 更新 Gateway Laya routing 测试，验证 `sdk_version` 和 `runtime`、calibration 身份以及 fallback 语义不变。
+- 对每个发布的 checkpoint 和受支持设备运行可选的真实模型 smoke/parity 测试，并与 NLaya/Python golden fixtures 对比。普通 CI 不下载模型。
+- 将 CI 中 Laya 和 routing-report 的 Python 测试步骤替换为 .NET 测试。确保 `tools/laya_service` 的构建、测试和文档命令均不需要 Python。
+
+默认以 JIT .NET 10 可执行程序运行服务。Gateway 的 NativeAOT 构建边界保持不变。未在受支持 TorchSharp 目标上成功发布并运行前，不得宣称服务支持 NativeAOT。
+
+## 文档与迁移
+
+更新 `docs/laya-routing.md`、其中文版、`tools/laya_service/README.md`、routing CLI 指南、CI workflow 以及所有报告命令引用。说明新的 package/runtime 要求、模型下载流程、本机命令格式、v2 校准重生成、设备支持、可选真实模型 smoke，以及 parity 并不能证明生产质量。保留现有 Laya 归属说明和 Apache-2.0 notice。只有在新实现及其测试覆盖到位后，才在实现过程中移除过时的 Python requirements、服务模块和测试。
diff --git a/docs/zh-CN/SITE_MAP.md b/docs/zh-CN/SITE_MAP.md
index 785036d0..ecd9cb99 100644
--- a/docs/zh-CN/SITE_MAP.md
+++ b/docs/zh-CN/SITE_MAP.md
@@ -49,6 +49,7 @@
| 集成 | Microsoft Teams | [TEAMS_SETUP.md](TEAMS_SETUP.md) |
| 集成 | WhatsApp | [WHATSAPP_SETUP.md](WHATSAPP_SETUP.md) |
| 集成 | A2A | [a2a.md](a2a.md) |
+| 集成 | 本地 Laya 决策路由 | [integrations/laya-routing.md](integrations/laya-routing.md) |
| 集成 | 外部编码后端 | [external-coding-backends.md](external-coding-backends.md) |
| 集成 | Nacos MCP Router（PoC） | [nacos-mcp-router.md](nacos-mcp-router.md) |
| 集成 | Nacos Live 验收与 Gateway 架构 | [nacos-live-architecture.md](nacos-live-architecture.md) |
@@ -132,6 +133,7 @@
Microsoft Teams
WhatsApp
A2A
+ 本地 Laya 决策路由
外部编码后端
Nacos MCP Router（PoC）
Tailscale 部署
diff --git a/docs/zh-CN/integrations/laya-routing.md b/docs/zh-CN/integrations/laya-routing.md
new file mode 100644
index 00000000..23cec1ed
--- /dev/null
+++ b/docs/zh-CN/integrations/laya-routing.md
@@ -0,0 +1,115 @@
+# 本地 Laya 决策路由
+
+OpenClaw 可将 Laya 作为可选的本地决策服务。Native 与 Microsoft Agent Framework runtime 共用决策路由策略；默认关闭。Laya 输出结构化决策，不是聊天模型，不应配置为 `Models.Profiles` 中的生成模型。
+
+## 运行边界
+
+独立服务以 .NET 10 构建，依赖 `NLaya`/`NLaya.TorchSharp` 1.0.0 和 `TorchSharp-cpu` 0.107.0。Gateway 仅通过 loopback HTTP 访问服务，不引用 NLaya 或 TorchSharp。当前按 JIT 服务部署；不声明 NativeAOT 支持。当前依赖集验证过 CPU。`cuda` 与 `mps` 只有在安装匹配的 TorchSharp backend 后才能启动；backend 不可用时服务会失败退出，不会静默切换到 CPU。
+
+Laya 由 Nandakishor Mukkunnoth（Nandakishor M）、ConvAI Innovations 和上游贡献者开发。模型、SDK 和研究成果归其作者所有。OpenClaw 单独维护本地服务、适配层和评估工具。详见[作者文章](https://laya.convaiinnovations.com/)、[源代码仓库](https://github.com/NandhaKishorM/laya)、[模型卡](https://huggingface.co/convaiinnovations/laya)以及[第三方声明](../../../tools/laya_service/THIRD_PARTY_NOTICES.md)。
+
+## 下载与启动
+
+在仓库根目录使用 .NET 10 SDK。默认 revision 固定为 `1c5edc17a7acd8701df6fc341c0d179f1c62c982`；可按需选择 checkpoint：
+
+```bash
+dotnet run --project tools/laya_service -c Release -- download \
+ --destination /path/to/laya-models \
+ --revision 1c5edc17a7acd8701df6fc341c0d179f1c62c982 \
+ --checkpoint english --checkpoint multilingual
+```
+
+下载器只获取 allowlist 中的模型、配置和 tokenizer 文件，为每个文件计算 SHA-256 并写入 `manifest.json`，同时复制上游 attribution 和 Apache-2.0 license。变更 revision 时使用新的目标目录。服务启动前会校验 manifest 路径与资产哈希；serve 过程只加载本地资产，不下载权重。模型和许可证仍属于上游项目，参见 [THIRD_PARTY_NOTICES.md](../../../tools/laya_service/THIRD_PARTY_NOTICES.md)。
+
+```bash
+dotnet run --project tools/laya_service -c Release -- serve \
+ --manifest /path/to/laya-models/manifest.json --device cpu --port 8099 --threads 4
+```
+
+端口默认 `8099`，设备默认 `cpu`，线程数默认 `4`。`--checkpoint auto` 为默认路由；`english` 与 `multilingual` 按语言路由。`typed-decisions` 必须下载后显式选择，不会因问题名称而自动启用。健康检查位于 `http://127.0.0.1:8099/health`。
+
+服务仅绑定 `127.0.0.1`，拒绝浏览器 Origin 和不匹配的 Host，限制请求大小和连接数，并且单次只允许一个推理请求，不排队。Gateway 绕过 proxy/redirect，只接受字面 loopback endpoint；Laya 失败时回到 baseline，不会 fallback 到托管 Jev。被选中的会话文本仍会跨越本地 HTTP 进程边界。
+
+## Gateway 配置
+
+保留现有 `Policy.Tiers` profile 映射，一次只启用一个决策 provider。Laya 示例：
+
+```json
+{
+ "OpenClaw": {
+ "DynamicTurnRouting": {
+ "Jev": { "Mode": "disabled" },
+ "Laya": {
+ "Mode": "shadow",
+ "Endpoint": "http://127.0.0.1:8099/v1/decisions",
+ "Model": "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982",
+ "CalibrationId": "",
+ "Language": "",
+ "TimeoutMs": 1500,
+ "MaxConcurrentRequests": 1,
+ "DiagnosticsPath": "routing/laya-decisions.jsonl"
+ }
+ }
+ }
+}
+```
+
+`shadow` 保持 baseline 结果不变，但会增加有界的评估耗时。服务 metadata 使用 `sdk_version: "1.0.0"` 和 `runtime: "NLaya"`；Gateway 会严格检查两项身份。错误、过载、超时和不确定结果都保留 baseline。共享策略仍保留脱敏、显式模型选择、安全 floor、sticky tier 和工具权限，不会授权执行操作。
+
+## 评估与校准
+
+评估集每行包含唯一 `case_id`、`state` 和覆盖每个问题的人工标签；省略 model/questions 时，工具使用共享 rubric 和固定 model：
+
+```json
+{"case_id":"cal-001","state":{"current_request":"将 hello 改成大写。","recent_conversation":[]},"labels":{"tier":"T0","high_risk":false,"requires_tools":false}}
+```
+
+准备代表性的拟合集和独立验证集，涵盖多语言、多轮引用、歧义、高影响任务和依赖工具的请求。每个 checkpoint/question-type/option-count bucket 至少 20 条只是工具下限，并非生产样本量建议。每个拟合桶需要多个标签；不要从模型输出来机械生成风险或工具标签。
+
+```bash
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/calibration-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/calibration-raw.jsonl
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/validation-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/validation-raw.jsonl
+dotnet run --project tools/laya_service -c Release -- calibrate \
+ --fit /path/to/calibration-raw.jsonl --validate /path/to/validation-raw.jsonl \
+ --output /path/to/calibration.json
+```
+
+`evaluate` 只允许字面 `http://127.0.0.1:PORT/v1/decisions`，禁用 proxy 与自动 redirect。Observation 保存 raw answer、标签、来源身份和 canonical request fingerprint，不包含 state；仍应按可能敏感的 metadata 保护这些文件。`calibrate` 按 checkpoint 与 `type:option-count` 分桶拟合温度，并检查 ID/fingerprint 不重叠、model/schema/runtime 一致和每桶样本要求。结果包括 held-out NLL、Brier、ECE、reliability bins 与 risk/coverage。产物是 UTF-8 JSON，末尾换行，`version: 2`，绑定模型 revision、schema hash、`sdk_version: "1.0.0"` 与 `runtime: "NLaya"`；命令输出产物的精确 SHA-256。
+
+以 `--calibration /path/to/calibration.json` 重启服务，并将打印的 SHA-256 配置到 `Laya.CalibrationId`。服务拒绝 v1 calibration、不匹配的 schema，或缺失的 checkpoint/bucket。先继续 shadow 并检查独立验证集，再考虑 active。温度缩放不能修复排序错误或知识缺失；仓库不提供生产 calibration artifact。
+
+## 路由报告
+
+Jev 与 Laya journal 使用同一个 provider-neutral .NET 报告命令。标签字段为 `decision_id`、`expected_tier`，以及可选 boolean `high_risk`。概率指标按 provider/model/rubric/checkpoint/revision/calibration/schema 分 cohort。无标签时不会声称准确率；PNG 图必须有带标签的概率数据。
+
+```bash
+dotnet run --project tools/laya_service -c Release -- report \
+ /path/to/jev-decisions.snapshot.jsonl \
+ --labels /path/to/tier-labels.jsonl --output /path/to/report.json \
+ --plot /path/to/reliability.png
+```
+
+## 回滚与验证
+
+将 `Laya.Mode` 设为 `disabled` 并重启 Gateway；也可以运行 `openclaw routing configure router --router disabled` 关闭 ONNX、Jev 和 Laya。删除可能重新启用路由的环境变量，并停止独立运行的 .NET 服务进程。
+
+```bash
+dotnet test tools/laya_service/tests/LayaService.Tests.csproj -c Release
+dotnet test src/OpenClaw.Tests/OpenClaw.Tests.csproj -c Release \
+ -p:OpenClawSkipDashboardBuild=true --filter FullyQualifiedName~LayaRoutingTests
+```
+
+这些自动化测试使用 fake 与 fixture，不会下载模型，也不证明真实模型质量。真实推理和 checkpoint parity 是需在已配置模型资产与设备 backend 的环境中单独执行的可选运维检查。Skill 选择、记忆重排和 workflow escalation 属于后续独立集成，需要各自的质量评估。
+
+可选的真实模型检查需先下载三个 checkpoint。分别用 `--checkpoint english`、`--checkpoint multilingual`、`--checkpoint typed-decisions` 启动服务；每个 checkpoint 都对同一份带标签 parity 数据集运行评估：
+
+```bash
+dotnet run --project tools/laya_service -c Release -- serve \
+ --manifest /path/to/laya-models/manifest.json --checkpoint english --device cpu
+dotnet run --project tools/laya_service -c Release -- evaluate /path/to/parity-cases.jsonl \
+ --endpoint http://127.0.0.1:8099/v1/decisions --output /path/to/english-observations.jsonl
+```
+
+对 `multilingual` 和 `typed-decisions` 重复执行，并使用不同的输出路径。将 observation 与经批准的参考输出比较：typed answer/category 应精确匹配；概率差异应在同一 device/runtime 下、运行前预先确定的容差内。记录 checkpoint、revision、runtime、device 和容差。仓库未提供真实模型 golden output 或生产验收阈值；此检查不属于普通 CI，也不构成生产路由质量证据。
\ No newline at end of file
diff --git a/scripts/evaluate-decision-routing.py b/scripts/evaluate-decision-routing.py
deleted file mode 100644
index 6974a87a..00000000
--- a/scripts/evaluate-decision-routing.py
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env python3
-"""Provider-neutral entry point; the previous Jev command remains supported."""
-from pathlib import Path
-import runpy
-
-if __name__ == "__main__":
- runpy.run_path(str(Path(__file__).with_name("evaluate-jev-routing.py")), run_name="__main__")
diff --git a/scripts/evaluate-jev-routing.py b/scripts/evaluate-jev-routing.py
deleted file mode 100644
index eed23a3c..00000000
--- a/scripts/evaluate-jev-routing.py
+++ /dev/null
@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-"""Summarize Jev or Laya routing journals; optional human labels measure tier quality, not task success."""
-import argparse
-from collections import Counter
-import json
-import math
-from pathlib import Path
-import sys
-
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from tools.laya_service.calibration import metrics
-
-
-TIERS = ("T0", "T1", "T2", "T3")
-
-
-def read_jsonl(path):
- with Path(path).open(encoding="utf-8") as stream:
- for line_number, line in enumerate(stream, 1):
- if line.strip():
- try:
- row = json.loads(line)
- except json.JSONDecodeError as exc:
- raise ValueError(f"{path}:{line_number}: invalid JSON") from exc
- if not isinstance(row, dict):
- raise ValueError(f"{path}:{line_number}: expected an object")
- yield row
-
-
-def percentile(values, fraction):
- ordered = sorted(values)
- return ordered[max(0, math.ceil(len(ordered) * fraction) - 1)] if ordered else None
-
-
-def quality(rows, labels, predict):
- confusion = {tier: {predicted: 0 for predicted in TIERS} for tier in TIERS}
- under = over = correct = high_risk_count = high_risk_retained = 0
- for row in rows:
- label = labels[row["decision_id"]]
- expected, predicted = label["expected_tier"], predict(row)
- confusion[expected][predicted] += 1
- correct += predicted == expected
- under += TIERS.index(predicted) < TIERS.index(expected)
- over += TIERS.index(predicted) > TIERS.index(expected)
- if label.get("high_risk", False):
- high_risk_count += 1
- high_risk_retained += TIERS.index(predicted) >= max(2, TIERS.index(expected))
- per_tier = {}
- for tier in TIERS:
- tp = confusion[tier][tier]
- fp = sum(confusion[other][tier] for other in TIERS if other != tier)
- fn = sum(confusion[tier][other] for other in TIERS if other != tier)
- per_tier[tier] = {
- "support": sum(confusion[tier].values()),
- "f1": 2 * tp / (2 * tp + fp + fn) if 2 * tp + fp + fn else 0,
- }
- return {
- "samples": len(rows), "accuracy": correct / len(rows),
- "under_routing_rate": under / len(rows), "over_routing_rate": over / len(rows),
- "high_risk_samples": high_risk_count,
- "high_risk_capability_retention": high_risk_retained / high_risk_count if high_risk_count else None,
- "macro_f1": sum(item["f1"] for item in per_tier.values()) / len(TIERS),
- "per_tier": per_tier, "confusion": confusion,
- }
-
-
-def summarize(rows, label_rows=()):
- if not rows:
- raise ValueError("The journal contains no decisions.")
- by_id = {}
- for row in rows:
- identifier = row.get("decision_id")
- if not identifier or identifier in by_id:
- raise ValueError("Every decision must have a unique nonempty decision_id.")
- if row.get("baseline_tier") not in TIERS or row.get("applied_tier") not in TIERS:
- raise ValueError(f"Invalid baseline/applied tier for {identifier}.")
- if row.get("proposed_tier") is not None and row["proposed_tier"] not in TIERS:
- raise ValueError(f"Invalid proposed tier for {identifier}.")
- latency = row.get("latency_ms")
- cost = row.get("estimated_cost_usd", 0)
- if not isinstance(latency, (int, float)) or not math.isfinite(latency) or latency < 0:
- raise ValueError(f"Invalid latency for {identifier}.")
- if cost is not None and (not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0):
- raise ValueError(f"Invalid cost for {identifier}.")
- by_id[identifier] = row
- proposed = [row for row in rows if row.get("proposed_tier") is not None]
- completed = [row for row in rows if row.get("input_tokens") is not None]
- latencies = [row["latency_ms"] for row in rows]
- report = {
- "decisions": len(rows), "responses_with_usage": len(completed),
- "eligible_proposals": len(proposed), "proposal_coverage": len(proposed) / len(rows),
- "modes": dict(Counter(row.get("mode", "unknown") for row in rows)),
- "providers": dict(Counter(row.get("provider", "jev") for row in rows)),
- "models": dict(Counter(row.get("model", "unreported") for row in rows)),
- "rubric_versions": dict(Counter(row.get("rubric_version", "unknown") for row in rows)),
- "reasons": dict(Counter(row.get("reason", "unknown") for row in rows)),
- "proposed_tiers": dict(Counter(row["proposed_tier"] for row in proposed)),
- "proposal_disagreement_with_baseline": sum(row["proposed_tier"] != row["baseline_tier"] for row in proposed) / len(proposed) if proposed else None,
- "added_latency_ms": {"p50": percentile(latencies, .50), "p95": percentile(latencies, .95), "max": max(latencies)},
- "reported_input_tokens": sum(row["input_tokens"] for row in completed),
- "estimated_reported_decision_cost_usd": round(sum(row.get("estimated_cost_usd") or 0 for row in rows), 8),
- "quality": None,
- "limitations": [
- "Decision cost excludes failed calls without usage, downstream models, retries, and cache effects.",
- "Tier labels do not measure task success or establish calibrated confidence.",
- "With ONNX disabled, baseline tier T2 is a bookkeeping default; the actual configured model is unchanged.",
- "The proposal includes confidence gates and safety floors; missing proposals fall back to the baseline.",
- "Compare model/rubric cohorts separately before tuning thresholds.",
- ],
- }
- labels = {}
- for label in label_rows:
- identifier = label.get("decision_id")
- if identifier not in by_id or identifier in labels or label.get("expected_tier") not in TIERS:
- raise ValueError("Labels must reference unique journal decision IDs and expected_tier T0 through T3.")
- if "high_risk" in label and not isinstance(label["high_risk"], bool):
- raise ValueError("high_risk must be a JSON boolean.")
- labels[identifier] = label
- if labels:
- labeled = [row for row in rows if row["decision_id"] in labels]
- report["quality"] = {
- "labeled_decisions": len(labeled), "label_coverage": len(labeled) / len(rows),
- "baseline": quality(labeled, labels, lambda row: row["baseline_tier"]),
- "always_t2": quality(labeled, labels, lambda row: "T2"),
- "jev_with_fallback": quality(labeled, labels, lambda row: row.get("proposed_tier") or row["baseline_tier"]),
- }
- # Evaluate raw tier distributions, separately from the safeguards/fallback policy.
- # Never pool calibration across checkpoint, rubric, or temperature artifacts.
- cohorts = {}
- for row in rows:
- if row["decision_id"] not in labels or not row.get("probabilities"):
- continue
- metadata = row.get("metadata") or {}
- identity = json.dumps([row.get("provider", "jev"), row.get("model"), row.get("rubric_version"),
- metadata.get("checkpoint"), metadata.get("revision"), metadata.get("calibration_id"),
- metadata.get("schema_hash")])
- cohorts.setdefault(identity, []).append({
- "answer": {"type": "choice", "probabilities": row["probabilities"]},
- "label": labels[row["decision_id"]]["expected_tier"]})
- report["calibration_quality"] = [{"cohort": json.loads(key), **metrics(values)} for key, values in cohorts.items()]
- if report["quality"]:
- report["quality"]["decision_with_fallback"] = report["quality"]["jev_with_fallback"]
- if any(row.get("provider") == "laya" for row in rows):
- del report["quality"]["jev_with_fallback"]
- return report
-
-
-def plot(report, path):
- import matplotlib
- matplotlib.use("Agg")
- import matplotlib.pyplot as plt
- cohorts = report["calibration_quality"]
- if not cohorts:
- raise ValueError("Reliability plots require labeled probabilities.")
- figure, axes = plt.subplots(len(cohorts), 2, figsize=(10, 4 * len(cohorts)), squeeze=False)
- for index, cohort in enumerate(cohorts):
- bins = cohort["reliability_bins"]
- axes[index, 0].plot([0, 1], [0, 1], "--", color="gray")
- axes[index, 0].plot([b["mean_top_probability"] for b in bins], [b["accuracy"] for b in bins], "o-")
- provider, model, rubric, checkpoint, revision, calibration_id, schema = cohort["cohort"]
- title = f"{provider} / {checkpoint or model or 'unknown'}\n{rubric or 'unknown rubric'} | revision {(revision or 'n/a')[:8]} | calibration {(calibration_id or 'n/a')[:8]}"
- axes[index, 0].set(xlabel="Mean top probability", ylabel="Accuracy", xlim=(0, 1), ylim=(0, 1))
- axes[index, 0].set_title(title, fontsize=9)
- curve = [point for point in cohort["risk_coverage"] if point["error_rate"] is not None]
- axes[index, 1].plot([p["coverage"] for p in curve], [p["error_rate"] for p in curve], "o-")
- axes[index, 1].set(xlabel="Coverage at entropy-confidence threshold", ylabel="Error rate", xlim=(0, 1), ylim=(0, 1))
- figure.tight_layout()
- figure.savefig(path, dpi=150)
- plt.close(figure)
-
-
-def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("journal", help="Path to a snapshot of jev-decisions.jsonl")
- parser.add_argument("--labels", help="Optional JSONL with decision_id, expected_tier, and high_risk")
- parser.add_argument("--output", help="Write report JSON here instead of stdout")
- parser.add_argument("--plot", help="Optional reliability/risk-coverage PNG (requires matplotlib)")
- args = parser.parse_args()
- try:
- report = summarize(list(read_jsonl(args.journal)), list(read_jsonl(args.labels)) if args.labels else [])
- if args.plot:
- plot(report, args.plot)
- rendered = json.dumps(report, indent=2, allow_nan=False) + "\n"
- if args.output:
- Path(args.output).write_text(rendered, encoding="utf-8")
- else:
- sys.stdout.write(rendered)
- except (OSError, ValueError, TypeError, KeyError, ImportError) as exc:
- parser.exit(2, f"{exc}\n")
-
-
-if __name__ == "__main__":
- main()
diff --git a/src/OpenClaw.Agent/Tools/XSearchTool.cs b/src/OpenClaw.Agent/Tools/XSearchTool.cs
index 0f2f43fc..057aa704 100644
--- a/src/OpenClaw.Agent/Tools/XSearchTool.cs
+++ b/src/OpenClaw.Agent/Tools/XSearchTool.cs
@@ -49,7 +49,7 @@ public async ValueTask ExecuteAsync(string argumentsJson, CancellationTo
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _bearerToken);
- var response = await _http.SendAsync(request, ct);
+ using var response = await _http.SendAsync(request, ct);
if (!response.IsSuccessStatusCode)
{
diff --git a/src/OpenClaw.Channels/DiscordChannel.cs b/src/OpenClaw.Channels/DiscordChannel.cs
index 45ff0443..3401d684 100644
--- a/src/OpenClaw.Channels/DiscordChannel.cs
+++ b/src/OpenClaw.Channels/DiscordChannel.cs
@@ -383,7 +383,7 @@ private async Task RegisterSlashCommandsAsync(CancellationToken ct)
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bot", _botToken);
request.Content = new StringContent(payload, Encoding.UTF8, "application/json");
- var response = await _http.SendAsync(request, ct);
+ using var response = await _http.SendAsync(request, ct);
if (response.IsSuccessStatusCode)
_logger.LogInformation("Registered Discord slash command '/{Command}'.", commandName);
else
diff --git a/src/OpenClaw.Channels/SlackChannel.cs b/src/OpenClaw.Channels/SlackChannel.cs
index 59dc7713..812514fd 100644
--- a/src/OpenClaw.Channels/SlackChannel.cs
+++ b/src/OpenClaw.Channels/SlackChannel.cs
@@ -61,7 +61,7 @@ public async ValueTask SendAsync(OutboundMessage outbound, CancellationToken ct)
request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", _botToken);
request.Content = JsonContent.Create(payload, SlackJsonContext.Default.SlackPostMessageRequest);
- var response = await _http.SendAsync(request, ct);
+ using var response = await _http.SendAsync(request, ct);
if ((int)response.StatusCode == 429)
{
diff --git a/src/OpenClaw.Gateway/BotFrameworkTokenValidator.cs b/src/OpenClaw.Gateway/BotFrameworkTokenValidator.cs
index d9b09b3f..655b575a 100644
--- a/src/OpenClaw.Gateway/BotFrameworkTokenValidator.cs
+++ b/src/OpenClaw.Gateway/BotFrameworkTokenValidator.cs
@@ -155,14 +155,14 @@ private async Task GetSigningKeysAsync(CancellationToken ct
if (_snapshot is { } snapshotLocked && snapshotLocked.ExpiresAt > DateTimeOffset.UtcNow)
return snapshotLocked;
- var metadataResponse = await _http.GetAsync(OpenIdMetadataUrl, ct);
+ using var metadataResponse = await _http.GetAsync(OpenIdMetadataUrl, ct);
metadataResponse.EnsureSuccessStatusCode();
await using var metadataStream = await metadataResponse.Content.ReadAsStreamAsync(ct);
using var metadataDocument = await JsonDocument.ParseAsync(metadataStream, cancellationToken: ct);
var jwksUrl = TryGetString(metadataDocument.RootElement, "jwks_uri");
jwksUrl = string.IsNullOrWhiteSpace(jwksUrl) ? DefaultJwksUrl : jwksUrl;
- var keysResponse = await _http.GetAsync(jwksUrl, ct);
+ using var keysResponse = await _http.GetAsync(jwksUrl, ct);
keysResponse.EnsureSuccessStatusCode();
await using var keysStream = await keysResponse.Content.ReadAsStreamAsync(ct);
using var keysDocument = await JsonDocument.ParseAsync(keysStream, cancellationToken: ct);
diff --git a/src/OpenClaw.Routing.Decisions/DecisionModels.cs b/src/OpenClaw.Routing.Decisions/DecisionModels.cs
index 3bb575fa..b3766673 100644
--- a/src/OpenClaw.Routing.Decisions/DecisionModels.cs
+++ b/src/OpenClaw.Routing.Decisions/DecisionModels.cs
@@ -41,6 +41,7 @@ public sealed class DecisionMetadata
public required string RubricVersion { get; init; }
public required string Device { get; init; }
public required string SdkVersion { get; init; }
+ public required string Runtime { get; init; }
public required bool Truncated { get; init; }
}
diff --git a/src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs b/src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs
index 46dc3029..c76da16e 100644
--- a/src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs
+++ b/src/OpenClaw.Routing.Decisions/LayaDecisionClient.cs
@@ -45,7 +45,7 @@ public async Task EvaluateAsync(DecisionRequest request, Cance
if (result.Model != _config.Model || metadata is null || metadata.Revision != _config.Model[5..] ||
metadata.Checkpoint is not ("english" or "multilingual" or "typed-decisions") ||
metadata.RubricVersion != request.RubricVersion || metadata.SchemaHash != expectedSchemaHash ||
- metadata.SdkVersion != "0.3.4" || metadata.Device is not ("cpu" or "cuda" or "mps"))
+ metadata.SdkVersion != "1.0.0" || metadata.Runtime != "NLaya" || metadata.Device is not ("cpu" or "cuda" or "mps"))
throw new DecisionException("laya_metadata_mismatch");
if (metadata.Truncated)
throw new DecisionException("laya_truncated_input");
diff --git a/src/OpenClaw.Tests/LayaRoutingTests.cs b/src/OpenClaw.Tests/LayaRoutingTests.cs
index 1be9b2a0..a16e6932 100644
--- a/src/OpenClaw.Tests/LayaRoutingTests.cs
+++ b/src/OpenClaw.Tests/LayaRoutingTests.cs
@@ -145,6 +145,7 @@ public async Task OversizedSerializedStateDoesNotReachServiceOrOpenCircuit(char
[InlineData("calibration_id", "laya_calibration_mismatch")]
[InlineData("schema_hash", "laya_metadata_mismatch")]
[InlineData("sdk_version", "laya_metadata_mismatch")]
+ [InlineData("runtime", "laya_metadata_mismatch")]
[InlineData("missing_metadata", "laya_metadata_mismatch")]
[InlineData("http_error", "http_503")]
public async Task UnusableLocalResponsesKeepBaselineWithoutHostedFallback(string corruption, string expected)
@@ -204,7 +205,7 @@ private sealed class Handler(LayaRoutingConfig config) : HttpMessageHandler
},
usage = new { input_tokens = 150, output_tokens = 0 },
metadata = new { checkpoint = "english", revision = config.Model[5..], calibration_id = Calibration,
- schema_hash = Calibration, rubric_version = "openclaw-laya-tiers-v1", device = "cpu", sdk_version = "0.3.4", truncated = false }
+ schema_hash = Calibration, rubric_version = "openclaw-laya-tiers-v1", device = "cpu", sdk_version = "1.0.0", runtime = "NLaya", truncated = false }
}))!.AsObject();
protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
diff --git a/tests/laya-service/test_service.py b/tests/laya-service/test_service.py
deleted file mode 100644
index c90e72cf..00000000
--- a/tests/laya-service/test_service.py
+++ /dev/null
@@ -1,238 +0,0 @@
-import copy
-import hashlib
-import json
-from pathlib import Path
-import sys
-import tempfile
-import threading
-import types
-import unittest
-from unittest.mock import patch
-from urllib.request import Request, urlopen
-from urllib.error import HTTPError
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
-from tools.laya_service import calibration, protocol, compat, runtime, download
-from tools.laya_service.__main__ import DecisionServer
-
-MODEL = 'laya@' + protocol.DEFAULT_REVISION
-QUESTIONS = {'tier': {'type': 'choice', 'instructions': 'Which task?', 'criteria': {'small': 'simple', 'large': 'complex'}}}
-
-
-def request():
- return {'model': MODEL, 'state': 'hello', 'questions': copy.deepcopy(QUESTIONS), 'rubric_version': 'test-v1'}
-
-
-def observation(identifier, label='small', probability=.99):
- return {'case_id': identifier, 'case_fingerprint': hashlib.sha256(identifier.encode()).hexdigest(), 'question_id': 'tier', 'checkpoint': 'english', 'model': MODEL,
- 'schema_hash': protocol.schema_hash(QUESTIONS), 'sdk_version': protocol.SDK_VERSION,
- 'source_calibration': 'raw', 'label': label,
- 'answer': {'type': 'choice', 'choice': 'small', 'probabilities': {'small': probability, 'large': 1-probability}}}
-
-
-class ProtocolTests(unittest.TestCase):
- def test_rubric_hash_matches_dotnet_golden_values(self):
- rubric = json.loads((Path(__file__).resolve().parents[2] /
- 'tools/laya_service/rubrics/openclaw-laya-tiers-v1.json').read_text())
- self.assertEqual('8c8de008c11734cf12b58c7914581ea2779c2e44356f3c5b1cf79df7fa953d9c',
- protocol.schema_hash(rubric['questions']))
- rubric['questions']['tier']['instructions'] = "Judge l'utilisateur & C++ café 中文 😀\u2028\u0001\n\t"
- self.assertEqual('2089e651a62138bf97aaf6c0ee38e4791540f7ec372064384cc7caf831b9da9c',
- protocol.schema_hash(rubric['questions']))
-
- def test_rejects_duplicate_nonfinite_and_invalid_contracts(self):
- for value in ['{"x":1,"x":2}', '{"x":NaN}']:
- with self.assertRaises(ValueError): protocol.read_json(value)
- for mutate in [lambda r: r.update(model='laya@latest'),
- lambda r: r.update(questions={}),
- lambda r: r['questions']['tier'].update(criteria={str(i): '' for i in range(21)}),
- lambda r: r.update(language='../model'),
- lambda r: r['questions']['tier'].update(type='text')]:
- value = request(); mutate(value)
- with self.assertRaises(protocol.Rejected): protocol.validate_request(value, MODEL)
- self.assertEqual(QUESTIONS, protocol.validate_request(request(), MODEL))
-
- def test_schema_identity_preserves_choice_order(self):
- changed = copy.deepcopy(QUESTIONS)
- changed['tier']['criteria'] = dict(reversed(list(changed['tier']['criteria'].items())))
- self.assertNotEqual(protocol.schema_hash(QUESTIONS), protocol.schema_hash(changed))
-
- def test_manifest_rejects_corruption_and_paths_outside_bundle(self):
- with tempfile.TemporaryDirectory() as tmp:
- root = Path(tmp); model = root/'model'; model.mkdir()
- hashes = {}
- for filename in protocol.MODEL_FILES:
- target = model/filename; target.parent.mkdir(exist_ok=True)
- target.write_text('test'); hashes[filename] = protocol.file_hash(target)
- manifest = root/'manifest.json'
- data = {'version': 1, 'revision': protocol.DEFAULT_REVISION,
- 'checkpoints': {'english': {'path': 'model', 'sha256': hashes}}}
- manifest.write_text(json.dumps(data))
- self.assertIn('english', runtime.load_manifest(manifest)['checkpoints'])
- (model/'model.safetensors').write_text('changed')
- with self.assertRaises(ValueError): runtime.load_manifest(manifest)
- data['checkpoints']['english']['path'] = '../outside'
- manifest.write_text(json.dumps(data))
- with self.assertRaises(ValueError): runtime.load_manifest(manifest)
-
- def test_download_selects_exact_files_and_pins_revision(self):
- calls = []
- def snapshot(repo, **kw):
- calls.append((repo, kw))
- for filename in kw['allow_patterns']:
- target = Path(kw['local_dir'])/filename; target.parent.mkdir(parents=True, exist_ok=True)
- target.write_text('{}' if filename.endswith('.json') else 'fake weight')
- hub = types.ModuleType('huggingface_hub'); hub.snapshot_download = snapshot
- with tempfile.TemporaryDirectory() as tmp, patch.dict(sys.modules, {'huggingface_hub': hub}):
- expected = {filename: hashlib.sha256(
- ('{}' if filename.endswith('.json') else 'fake weight').encode()).hexdigest()
- for filename in protocol.MODEL_FILES}
- with patch.object(download, 'DEFAULT_FILE_HASHES', {'english': expected}):
- manifest = download.prepare(tmp, protocol.DEFAULT_REVISION, ['english'])
- self.assertEqual(list(protocol.MODEL_FILES), calls[0][1]['allow_patterns'])
- self.assertEqual(protocol.DEFAULT_REVISION, calls[0][1]['revision'])
- self.assertIn('english', runtime.load_manifest(manifest)['checkpoints'])
- self.assertTrue((Path(tmp)/'LAYA-NOTICE.md').exists())
- with self.assertRaises(ValueError): download.prepare(tmp, 'latest', ['english'])
- with tempfile.TemporaryDirectory() as tmp, patch.dict(sys.modules, {'huggingface_hub': hub}), \
- patch.object(download, 'DEFAULT_FILE_HASHES', {'english': {**expected, 'model.safetensors': '0'*64}}):
- with self.assertRaisesRegex(ValueError, 'upstream digest'):
- download.prepare(tmp, protocol.DEFAULT_REVISION, ['english'])
-
-
-class CompatibilityTests(unittest.TestCase):
- def test_armenian_minority_and_unknown_scripts_never_use_english(self):
- lang = types.ModuleType('laya.lang')
- lang.analyse = lambda state: {'is_english': True}
- common = types.ModuleType('laya.common')
- common.serialize_state = lambda state: state if isinstance(state, str) else str(state)
- with patch.dict(sys.modules, {'laya.lang': lang, 'laya.common': common}):
- for text in ['Հայերեն', 'English text with Հայերեն', 'हिन्दी', 'ქართული']:
- self.assertEqual('multilingual', compat.select_checkpoint(text))
- with self.assertRaises(protocol.Rejected): compat.select_checkpoint(text, 'en')
- self.assertEqual('english', compat.select_checkpoint('Hello!'))
- self.assertEqual('multilingual', compat.select_checkpoint('Bonjour', 'fr'))
-
- def test_token_budgets_fail_before_sdk_truncation(self):
- common = types.ModuleType('laya.common')
- common.render_options = lambda q: list(q['crit'].values())
- common.serialize_state = str
- class Tokenizer:
- mask_token = '[MASK]'
- def __call__(self, text, **kw): return {'input_ids': list(text)}
- agent = types.SimpleNamespace(tok=Tokenizer(), cfg={'max_len': 90, 'head_max_len': 50},
- _to_internal=lambda q: {'t': q['type'], 'ins': q['instructions'], 'crit': q['criteria']})
- with patch.dict(sys.modules, {'laya.common': common}):
- compat.ensure_complete(agent, 'hello', QUESTIONS)
- with self.assertRaisesRegex(protocol.Rejected, 'state_too_long'):
- compat.ensure_complete(agent, 'x'*100, QUESTIONS)
- long_question = copy.deepcopy(QUESTIONS); long_question['tier']['instructions'] = 'x'*60
- with self.assertRaisesRegex(protocol.Rejected, 'question_too_long'):
- compat.ensure_complete(agent, 'hello', long_question)
- long_option = copy.deepcopy(QUESTIONS); long_option['tier']['criteria']['small'] = 'x'*60
- with self.assertRaisesRegex(protocol.Rejected, 'option_too_long'):
- compat.ensure_complete(agent, 'hello', long_option)
- with self.assertRaisesRegex(protocol.Rejected, 'reserved_token'):
- compat.ensure_complete(agent, '[MASK]', QUESTIONS)
-
-
-class CalibrationTests(unittest.TestCase):
- def test_fits_and_measures_disjoint_held_out_cases(self):
- train = [observation('train'+str(i), 'small' if i%2 else 'large') for i in range(40)]
- test = [observation('test'+str(i), 'small' if i%2 else 'large') for i in range(40)]
- artifact = calibration.fit(train, test)
- results = artifact['validation']['english']['choice:2']
- self.assertLess(results['calibrated']['nll'], results['raw']['nll'])
- self.assertLess(results['calibrated']['ece'], results['raw']['ece'])
- self.assertEqual(40, results['calibrated']['samples'])
- with tempfile.TemporaryDirectory() as tmp:
- path = Path(tmp)/'calibration.json'; path.write_text(json.dumps(artifact))
- fitted = calibration.Calibration(path, MODEL)
- fitted.check(protocol.schema_hash(QUESTIONS), 'english', QUESTIONS)
- with self.assertRaises(protocol.Rejected): fitted.check('a'*64, 'english', QUESTIONS)
- with self.assertRaises(protocol.Rejected): fitted.check(protocol.schema_hash(QUESTIONS), 'multilingual', QUESTIONS)
- self.assertEqual(protocol.file_hash(path), fitted.identifier)
-
- def test_rejects_leakage_mixed_versions_and_sparse_data(self):
- rows = [observation(str(i), 'small' if i%2 else 'large') for i in range(20)]
- with self.assertRaisesRegex(ValueError, 'overlap'): calibration.fit(rows, rows)
- with self.assertRaises(ValueError): calibration.fit(rows[:2], [observation('test')])
- other = [dict(r, case_id='test'+r['case_id'], schema_hash='a'*64) for r in rows]
- with self.assertRaises(ValueError): calibration.fit(rows, other)
- other = [dict(r, case_id='renamed'+r['case_id']) for r in rows]
- with self.assertRaisesRegex(ValueError, 'identical requests'): calibration.fit(rows, other)
-
- def test_metrics_include_abstention_errors_and_ties(self):
- rows = [observation('one'), observation('two', 'large')]
- result = calibration.metrics(rows)
- self.assertEqual(.5, result['accuracy'])
- self.assertEqual(2, result['samples'])
- self.assertTrue(all(x['coverage'] in (0, 1) for x in result['risk_coverage']))
- score = {'type': 'score', 'probabilities': {'0': .1, '1': .2, '2': .7}, 'score': 1.6}
- fixed = calibration.transform(score, 2)
- self.assertAlmostEqual(1, sum(fixed['probabilities'].values()))
- self.assertLess(fixed['score'], score['score'])
- self.assertAlmostEqual(.8, calibration.transform({'type': 'noul', 'noul': .8}, 1)['noul'])
-
- def test_rejects_boolean_probabilities(self):
- with self.assertRaisesRegex(protocol.Rejected, 'invalid_model_probabilities'):
- calibration.transform({'type': 'noul', 'noul': True}, 1)
-
-
-class ServerTests(unittest.TestCase):
- def setUp(self):
- self.calls = 0
- self.failure = None
- owner = self
- class FakeRuntime:
- def health(self): return {'ready': True}
- def predict(self, value):
- owner.calls += 1
- protocol.validate_request(value, MODEL)
- if owner.failure:
- raise owner.failure
- return {'ok': True}
- self.server = DecisionServer(0, FakeRuntime())
- self.thread = threading.Thread(target=self.server.serve_forever, daemon=True); self.thread.start()
- self.url = f'http://127.0.0.1:{self.server.server_port}'
-
- def tearDown(self):
- self.server.shutdown(); self.server.server_close(); self.thread.join()
-
- def post(self, value, **headers):
- return urlopen(Request(self.url+'/v1/decisions', data=json.dumps(value).encode(),
- headers={'Content-Type': 'application/json', **headers}), timeout=2)
-
- def test_health_and_valid_request(self):
- with urlopen(self.url+'/health') as response: self.assertTrue(json.load(response)['ready'])
- with self.post(request()) as response: self.assertTrue(json.load(response)['ok'])
- self.assertEqual(1, self.calls)
-
- def test_browser_origins_bad_hosts_overload_and_large_body(self):
- for headers in [{'Origin': 'https://example.com'}, {'Host': 'attacker.invalid'}]:
- with self.assertRaises(HTTPError) as error: self.post(request(), **headers)
- self.assertEqual(403, error.exception.code); error.exception.close()
- with self.server.inference:
- with self.assertRaises(HTTPError) as error: self.post(request())
- self.assertEqual(503, error.exception.code); error.exception.close()
- with self.assertRaises(HTTPError) as error: self.post({'state': 'x'*70000})
- self.assertEqual(413, error.exception.code); error.exception.close()
- self.assertEqual(0, self.calls)
-
- def test_errors_do_not_echo_request_contents(self):
- value = request(); value['model'] = 'SECRET'
- with self.assertRaises(HTTPError) as error: self.post(value)
- self.assertEqual(422, error.exception.code)
- self.assertNotIn(b'SECRET', error.exception.read()); error.exception.close()
-
- def test_runtime_failures_are_service_errors_not_client_errors(self):
- self.failure = ValueError('invalid model output')
- with self.assertRaises(HTTPError) as error:
- self.post(request())
- self.assertEqual(503, error.exception.code)
- self.assertEqual({'error': 'inference_failed'}, json.load(error.exception))
- error.exception.close()
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/tests/routing-eval/test_jev_report.py b/tests/routing-eval/test_jev_report.py
deleted file mode 100644
index 75463ab6..00000000
--- a/tests/routing-eval/test_jev_report.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import importlib.util
-from pathlib import Path
-import unittest
-
-spec = importlib.util.spec_from_file_location("jev_report", Path(__file__).resolve().parents[2] / "scripts/evaluate-jev-routing.py")
-module = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(module)
-
-
-class JevReportTests(unittest.TestCase):
- def test_labels_compare_effective_policy_including_fallback(self):
- rows = [
- {"decision_id": "a", "baseline_tier": "T2", "applied_tier": "T2", "proposed_tier": "T0", "latency_ms": 100, "input_tokens": 2000, "estimated_cost_usd": .000084},
- {"decision_id": "b", "baseline_tier": "T3", "applied_tier": "T3", "reason": "timeout", "latency_ms": 1500},
- ]
- labels = [{"decision_id": "a", "expected_tier": "T0"}, {"decision_id": "b", "expected_tier": "T3", "high_risk": True}]
- report = module.summarize(rows, labels)
- self.assertEqual(1, report["quality"]["jev_with_fallback"]["accuracy"])
- self.assertEqual(.5, report["quality"]["baseline"]["accuracy"])
- self.assertEqual(.5, report["quality"]["always_t2"]["under_routing_rate"])
- self.assertEqual(1, report["quality"]["jev_with_fallback"]["high_risk_capability_retention"])
- self.assertEqual(.000084, report["estimated_reported_decision_cost_usd"])
- self.assertEqual(1500, report["added_latency_ms"]["p95"])
-
- def test_calibration_is_split_by_checkpoint_and_counts_abstention(self):
- rows = [{"decision_id": str(i), "baseline_tier": "T2", "applied_tier": "T2", "latency_ms": 1,
- "provider": "laya", "model": "laya@revision", "rubric_version": "v1",
- "metadata": {"checkpoint": checkpoint, "calibration_id": "raw"},
- "probabilities": {"T0": .1, "T1": .1, "T2": .1, "T3": .1, "abstain": .6}}
- for i, checkpoint in enumerate(["english", "multilingual"])]
- labels = [{"decision_id": str(i), "expected_tier": "T0"} for i in range(2)]
- report = module.summarize(rows, labels)
- self.assertEqual(2, len(report["calibration_quality"]))
- self.assertEqual(0, report["calibration_quality"][0]["accuracy"])
- self.assertIn("decision_with_fallback", report["quality"])
- self.assertNotIn("jev_with_fallback", report["quality"])
-
- def test_unlabeled_data_does_not_claim_accuracy(self):
- report = module.summarize([{"decision_id": "a", "baseline_tier": "T2", "applied_tier": "T2", "latency_ms": 0}])
- self.assertIsNone(report["quality"])
- self.assertIsNone(report["proposal_disagreement_with_baseline"])
-
- def test_rejects_empty_duplicate_or_unmatched_data(self):
- row = {"decision_id": "a", "baseline_tier": "T2", "applied_tier": "T2", "latency_ms": 1}
- with self.assertRaises(ValueError):
- module.summarize([])
- with self.assertRaises(ValueError):
- module.summarize([row, row])
- with self.assertRaises(ValueError):
- module.summarize([row], [{"decision_id": "unknown", "expected_tier": "T0"}])
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/tools/laya_service/CommandLine.cs b/tools/laya_service/CommandLine.cs
new file mode 100644
index 00000000..523b0536
--- /dev/null
+++ b/tools/laya_service/CommandLine.cs
@@ -0,0 +1,93 @@
+namespace OpenClaw.LayaService;
+
+public sealed record CommandInvocation(
+ string Command,
+ CommandOptions Options,
+ IReadOnlyList Arguments);
+
+public sealed class CommandOptions
+{
+ private readonly IReadOnlyDictionary> _values;
+
+ internal CommandOptions(Dictionary> values)
+ {
+ _values = values.ToDictionary(
+ pair => pair.Key,
+ pair => (IReadOnlyList)pair.Value.ToArray(),
+ StringComparer.Ordinal);
+ }
+
+ public bool Contains(string name) => _values.ContainsKey(name);
+
+ public string? Get(string name)
+ => _values.TryGetValue(name, out var values) ? values[0] : null;
+
+ public IReadOnlyList GetMany(string name)
+ => _values.TryGetValue(name, out var values) ? values : Array.Empty();
+}
+
+public static class CommandLine
+{
+ private static readonly IReadOnlyDictionary> OptionsByCommand =
+ new Dictionary>(StringComparer.Ordinal)
+ {
+ ["serve"] = ["manifest", "calibration", "port", "device", "checkpoint", "threads"],
+ ["download"] = ["destination", "revision", "checkpoint"],
+ ["evaluate"] = ["endpoint", "output"],
+ ["calibrate"] = ["fit", "validate", "output"],
+ ["report"] = ["labels", "output", "plot"]
+ };
+
+ public static CommandInvocation Parse(string[] args)
+ {
+ ArgumentNullException.ThrowIfNull(args);
+ if (args.Length == 0 || !OptionsByCommand.TryGetValue(args[0], out var allowedOptions))
+ {
+ throw new ArgumentException("Invalid command line.", nameof(args));
+ }
+
+ var command = args[0];
+ var values = new Dictionary>(StringComparer.Ordinal);
+ var positionalArguments = new List();
+
+ for (var index = 1; index < args.Length; index++)
+ {
+ var argument = args[index];
+ if (!argument.StartsWith("--", StringComparison.Ordinal))
+ {
+ if (command is not ("evaluate" or "report") || positionalArguments.Count != 0)
+ {
+ throw new ArgumentException("Invalid command line.", nameof(args));
+ }
+
+ positionalArguments.Add(argument);
+ continue;
+ }
+
+ var name = argument[2..];
+ if (name.Length == 0 || !allowedOptions.Contains(name))
+ {
+ throw new ArgumentException("Invalid command line.", nameof(args));
+ }
+
+ if (index + 1 >= args.Length || args[index + 1].StartsWith("--", StringComparison.Ordinal))
+ {
+ throw new ArgumentException("Invalid command line.", nameof(args));
+ }
+
+ if (!values.TryGetValue(name, out var optionValues))
+ {
+ optionValues = [];
+ values.Add(name, optionValues);
+ }
+ else if (name != "checkpoint")
+ {
+ throw new ArgumentException("Invalid command line.", nameof(args));
+ }
+
+ optionValues.Add(args[++index]);
+ }
+
+ return new CommandInvocation(command, new CommandOptions(values), positionalArguments.ToArray());
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Evaluation/AnswerDistribution.cs b/tools/laya_service/Evaluation/AnswerDistribution.cs
new file mode 100644
index 00000000..8e5668a7
--- /dev/null
+++ b/tools/laya_service/Evaluation/AnswerDistribution.cs
@@ -0,0 +1,42 @@
+using System.Text.Json;
+
+namespace OpenClaw.LayaService.Evaluation;
+
+internal sealed record AnswerDistribution(string Type, IReadOnlyList Keys, IReadOnlyList Probabilities)
+{
+ public static AnswerDistribution Parse(JsonElement answer, JsonElement question)
+ {
+ if (answer.ValueKind != JsonValueKind.Object || question.ValueKind != JsonValueKind.Object)
+ throw new InvalidDataException("Invalid answer distribution.");
+ var type = question.GetProperty("type").GetString();
+ if (answer.GetProperty("type").GetString() != type)
+ throw new InvalidDataException("Answer type mismatch.");
+
+ if (type == "noul")
+ {
+ var value = answer.GetProperty("noul");
+ if (value.ValueKind != JsonValueKind.Number || !value.TryGetDouble(out var probability) || !double.IsFinite(probability) || probability is < 0 or > 1)
+ throw new InvalidDataException("Invalid answer probability.");
+ return new AnswerDistribution(type, ["false", "true"], [1 - probability, probability]);
+ }
+
+ if (type is not ("choice" or "score") || !answer.TryGetProperty("probabilities", out var distribution) || distribution.ValueKind != JsonValueKind.Object)
+ throw new InvalidDataException("Invalid answer distribution.");
+ var keys = type == "choice"
+ ? question.GetProperty("criteria").EnumerateObject().Select(candidate => candidate.Name).ToArray()
+ : Enumerable.Range(0, question.GetProperty("criteria").GetArrayLength()).Select(index => index.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToArray();
+ if (keys.Length < 2 || distribution.GetPropertyCount() != keys.Length)
+ throw new InvalidDataException("Invalid answer distribution.");
+ var probabilities = new double[keys.Length];
+ for (var index = 0; index < keys.Length; index++)
+ {
+ if (!distribution.TryGetProperty(keys[index], out var item) || item.ValueKind != JsonValueKind.Number ||
+ !item.TryGetDouble(out probabilities[index]) || !double.IsFinite(probabilities[index]) || probabilities[index] is < 0 or > 1)
+ throw new InvalidDataException("Invalid answer probability.");
+ }
+ var sum = probabilities.Sum();
+ if (Math.Abs(sum - 1) > 0.002) throw new InvalidDataException("Invalid answer probability sum.");
+ for (var index = 0; index < probabilities.Length; index++) probabilities[index] /= sum;
+ return new AnswerDistribution(type, keys, probabilities);
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Evaluation/CalibrationFitter.cs b/tools/laya_service/Evaluation/CalibrationFitter.cs
new file mode 100644
index 00000000..2e635bf7
--- /dev/null
+++ b/tools/laya_service/Evaluation/CalibrationFitter.cs
@@ -0,0 +1,258 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using OpenClaw.LayaService.Inference;
+using OpenClaw.LayaService.Models;
+
+namespace OpenClaw.LayaService.Evaluation;
+
+public sealed record CalibrationArtifact
+{
+ [JsonPropertyName("version")]
+ public int Version { get; init; } = 2;
+
+ [JsonPropertyName("model")]
+ public required string Model { get; init; }
+
+ [JsonPropertyName("schema_hash")]
+ public required string SchemaHash { get; init; }
+
+ [JsonPropertyName("sdk_version")]
+ public required string SdkVersion { get; init; }
+
+ [JsonPropertyName("runtime")]
+ public string Runtime { get; init; } = "NLaya";
+
+ [JsonPropertyName("temperatures")]
+ public required SortedDictionary> Temperatures { get; init; }
+
+ [JsonPropertyName("validation")]
+ public required SortedDictionary> Validation { get; init; }
+}
+
+public sealed record CalibrationBucketValidation(
+ [property: JsonPropertyName("calibration_samples")] int CalibrationSamples,
+ [property: JsonPropertyName("raw")] CalibrationMetricsResult Raw,
+ [property: JsonPropertyName("calibrated")] CalibrationMetricsResult Calibrated);
+
+public static class CalibrationFitter
+{
+ private static readonly JsonSerializerOptions ArtifactJson = new() { WriteIndented = true };
+ private static readonly HashSet ObservationProperties =
+ ["case_id", "case_fingerprint", "question_id", "model", "checkpoint", "schema_hash", "sdk_version", "runtime", "source_calibration", "answer", "label"];
+
+ public static async Task> ReadObservationsAsync(string path, CancellationToken cancellationToken)
+ {
+ var observations = new List();
+ using var reader = new StreamReader(path, System.Text.Encoding.UTF8, detectEncodingFromByteOrderMarks: false);
+ while (await reader.ReadLineAsync(cancellationToken) is { } line)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (string.IsNullOrWhiteSpace(line)) continue;
+ using var document = JsonDocument.Parse(line, new JsonDocumentOptions { MaxDepth = 32 });
+ RejectDuplicateProperties(document.RootElement);
+ if (document.RootElement.ValueKind != JsonValueKind.Object ||
+ document.RootElement.EnumerateObject().Any(property => !ObservationProperties.Contains(property.Name)) ||
+ document.RootElement.TryGetProperty("state", out _))
+ {
+ throw new InvalidDataException("Observation contains unsupported fields.");
+ }
+ observations.Add(document.RootElement.Deserialize() ?? throw new InvalidDataException("Invalid observation row."));
+ }
+ return observations;
+ }
+
+ public static CalibrationArtifact Fit(IReadOnlyList training, IReadOnlyList validation, int minimumSamples = 20)
+ {
+ ArgumentNullException.ThrowIfNull(training);
+ ArgumentNullException.ThrowIfNull(validation);
+ if (minimumSamples < 1 || training.Count == 0 || validation.Count == 0) throw new InvalidDataException("Calibration datasets must be nonempty.");
+ ValidateObservationSet(training);
+ ValidateObservationSet(validation);
+ if (training.Select(row => row.CaseId).Intersect(validation.Select(row => row.CaseId), StringComparer.Ordinal).Any() ||
+ training.Select(row => row.CaseFingerprint).Intersect(validation.Select(row => row.CaseFingerprint), StringComparer.Ordinal).Any())
+ {
+ throw new InvalidDataException("Calibration and validation cases must be disjoint.");
+ }
+
+ var identities = training.Concat(validation)
+ .Select(row => (row.Model, row.SchemaHash, row.RuntimeVersion, row.Runtime))
+ .Distinct()
+ .ToArray();
+ if (identities.Length != 1) throw new InvalidDataException("Use one model, schema and runtime per calibration artifact.");
+ var (model, schemaHash, runtimeVersion, runtime) = identities[0];
+ if (!model.StartsWith("laya@", StringComparison.Ordinal) || !ModelManifest.IsRevision(model[5..]) ||
+ !ModelManifest.IsHash(schemaHash) || runtimeVersion != NLayaDecisionPredictor.SdkVersion || runtime != "NLaya")
+ {
+ throw new InvalidDataException("Calibration provenance is not pinned to this NLaya runtime.");
+ }
+
+ var trainingGroups = Group(training);
+ var validationGroups = Group(validation);
+ if (!trainingGroups.Keys.ToHashSet().SetEquals(validationGroups.Keys))
+ throw new InvalidDataException("Training and validation must cover the same calibration buckets.");
+
+ var temperatures = new SortedDictionary>(StringComparer.Ordinal);
+ var validationMetrics = new SortedDictionary>(StringComparer.Ordinal);
+ foreach (var group in trainingGroups.Keys.OrderBy(key => key.Checkpoint, StringComparer.Ordinal).ThenBy(key => key.Bucket, StringComparer.Ordinal))
+ {
+ var train = trainingGroups[group];
+ var test = validationGroups[group];
+ if (train.Count < minimumSamples || test.Count < minimumSamples || train.Select(row => row.Prediction.Label).Distinct(StringComparer.Ordinal).Count() < 2)
+ throw new InvalidDataException("Each bucket needs the minimum samples on both splits and multiple training labels.");
+
+ var temperature = FitTemperature(train);
+ var rawMetrics = CalibrationMetrics.Measure(test.Select(row => row.Prediction).ToArray());
+ var calibratedPredictions = test.Select(row => row.Prediction with
+ {
+ Probabilities = Scale(row.Prediction.Probabilities, temperature)
+ }).ToArray();
+ var calibratedMetrics = CalibrationMetrics.Measure(calibratedPredictions);
+ if (!temperatures.TryGetValue(group.Checkpoint, out var checkpointTemperatures))
+ {
+ checkpointTemperatures = new SortedDictionary(StringComparer.Ordinal);
+ temperatures.Add(group.Checkpoint, checkpointTemperatures);
+ validationMetrics.Add(group.Checkpoint, new SortedDictionary(StringComparer.Ordinal));
+ }
+ checkpointTemperatures.Add(group.Bucket, temperature);
+ validationMetrics[group.Checkpoint].Add(group.Bucket, new CalibrationBucketValidation(train.Count, rawMetrics, calibratedMetrics));
+ }
+
+ return new CalibrationArtifact
+ {
+ Model = model,
+ SchemaHash = schemaHash,
+ SdkVersion = runtimeVersion,
+ Runtime = runtime,
+ Temperatures = temperatures,
+ Validation = validationMetrics
+ };
+ }
+
+ public static string Serialize(CalibrationArtifact artifact)
+ => JsonSerializer.Serialize(artifact, ArtifactJson) + "\n";
+
+ public static async Task WriteAsync(CalibrationArtifact artifact, string outputPath, CancellationToken cancellationToken)
+ {
+ var payload = System.Text.Encoding.UTF8.GetBytes(Serialize(artifact));
+ var fullPath = Path.GetFullPath(outputPath);
+ var directory = Path.GetDirectoryName(fullPath) ?? throw new ArgumentException("Invalid output path.");
+ Directory.CreateDirectory(directory);
+ var temporary = Path.Combine(directory, "." + Path.GetFileName(fullPath) + "." + Guid.NewGuid().ToString("N") + ".tmp");
+ try
+ {
+ await using (var stream = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096,
+ FileOptions.Asynchronous | FileOptions.WriteThrough))
+ {
+ await stream.WriteAsync(payload, cancellationToken);
+ await stream.FlushAsync(cancellationToken);
+ stream.Flush(flushToDisk: true);
+ }
+ File.Move(temporary, fullPath, overwrite: true);
+ return Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(payload));
+ }
+ finally
+ {
+ if (File.Exists(temporary)) File.Delete(temporary);
+ }
+ }
+
+ private static void ValidateObservationSet(IReadOnlyList observations)
+ {
+ var uniqueQuestionRows = new HashSet<(string CaseId, string QuestionId)>();
+ foreach (var row in observations)
+ {
+ if (string.IsNullOrWhiteSpace(row.CaseId) || string.IsNullOrWhiteSpace(row.QuestionId) ||
+ !ModelManifest.IsHash(row.CaseFingerprint) || !ModelManifest.IsHash(row.SchemaHash) ||
+ !ModelManifest.CheckpointNames.Contains(row.Checkpoint) || row.SourceCalibration != "raw" || row.Runtime != "NLaya" ||
+ !uniqueQuestionRows.Add((row.CaseId, row.QuestionId)))
+ {
+ throw new InvalidDataException("Invalid or duplicate raw observation.");
+ }
+ _ = ToPrediction(row);
+ }
+ }
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name)) throw new InvalidDataException("Duplicate observation property.");
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray()) RejectDuplicateProperties(item);
+ }
+ }
+
+ private static Dictionary<(string Checkpoint, string Bucket), List<(Observation Observation, LabeledPrediction Prediction)>> Group(
+ IReadOnlyList observations)
+ {
+ var groups = new Dictionary<(string Checkpoint, string Bucket), List<(Observation, LabeledPrediction)>>();
+ foreach (var row in observations)
+ {
+ var prediction = ToPrediction(row);
+ var key = (row.Checkpoint, prediction.AnswerType + ":" + prediction.CandidateKeys.Count.ToString(System.Globalization.CultureInfo.InvariantCulture));
+ if (!groups.TryGetValue(key, out var values)) groups.Add(key, values = []);
+ values.Add((row, prediction));
+ }
+ return groups;
+ }
+
+ private static LabeledPrediction ToPrediction(Observation row)
+ {
+ using var questionDocument = JsonDocument.Parse("{}");
+ var answer = row.RawAnswer;
+ var type = answer.GetProperty("type").GetString() ?? throw new InvalidDataException("Invalid raw answer.");
+ if (type == "noul")
+ {
+ var probability = answer.GetProperty("noul").GetDouble();
+ return CalibrationMetrics.Normalize(new LabeledPrediction(type, ["false", "true"], [1 - probability, probability], row.Label));
+ }
+
+ var probabilities = answer.GetProperty("probabilities");
+ var keys = probabilities.EnumerateObject().Select(item => item.Name).ToArray();
+ var values = probabilities.EnumerateObject().Select(item =>
+ {
+ if (item.Value.ValueKind != JsonValueKind.Number) throw new InvalidDataException("Invalid raw answer probability.");
+ return item.Value.GetDouble();
+ }).ToArray();
+ return CalibrationMetrics.Normalize(new LabeledPrediction(type, keys, values, row.Label));
+ }
+
+ private static double FitTemperature(IReadOnlyList<(Observation Observation, LabeledPrediction Prediction)> rows)
+ {
+ var bestTemperature = 0.1;
+ var bestLoss = double.PositiveInfinity;
+ for (var index = 0; index <= 160; index++)
+ {
+ var temperature = Math.Pow(10, -1 + index / 80.0);
+ var loss = 0.0;
+ foreach (var row in rows)
+ {
+ var prediction = row.Prediction;
+ var expectedIndex = CalibrationMetrics.IndexOfLabel(prediction.CandidateKeys, prediction.Label);
+ loss -= Math.Log(Math.Max(Scale(prediction.Probabilities, temperature)[expectedIndex], 1e-9));
+ }
+ if (loss < bestLoss)
+ {
+ bestLoss = loss;
+ bestTemperature = temperature;
+ }
+ }
+ return bestTemperature;
+ }
+
+ private static double[] Scale(IReadOnlyList probabilities, double temperature)
+ {
+ var logits = probabilities.Select(probability => Math.Log(Math.Max(probability, 1e-9)) / temperature).ToArray();
+ var peak = logits.Max();
+ var values = logits.Select(value => Math.Exp(value - peak)).ToArray();
+ var sum = values.Sum();
+ return values.Select(value => value / sum).ToArray();
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Evaluation/CalibrationMetrics.cs b/tools/laya_service/Evaluation/CalibrationMetrics.cs
new file mode 100644
index 00000000..9e5e65d9
--- /dev/null
+++ b/tools/laya_service/Evaluation/CalibrationMetrics.cs
@@ -0,0 +1,119 @@
+using System.Text.Json.Serialization;
+
+namespace OpenClaw.LayaService.Evaluation;
+
+public sealed record LabeledPrediction(
+ string AnswerType,
+ IReadOnlyList CandidateKeys,
+ IReadOnlyList Probabilities,
+ string Label);
+
+public sealed record ReliabilityBin(
+ [property: JsonPropertyName("lower")] double Lower,
+ [property: JsonPropertyName("upper")] double Upper,
+ [property: JsonPropertyName("samples")] int Samples,
+ [property: JsonPropertyName("mean_top_probability")] double MeanTopProbability,
+ [property: JsonPropertyName("accuracy")] double Accuracy);
+
+public sealed record RiskCoveragePoint(
+ [property: JsonPropertyName("confidence_threshold")] double ConfidenceThreshold,
+ [property: JsonPropertyName("coverage")] double Coverage,
+ [property: JsonPropertyName("error_rate")] double? ErrorRate);
+
+public sealed record CalibrationMetricsResult(
+ [property: JsonPropertyName("samples")] int Samples,
+ [property: JsonPropertyName("accuracy")] double Accuracy,
+ [property: JsonPropertyName("nll")] double Nll,
+ [property: JsonPropertyName("brier")] double Brier,
+ [property: JsonPropertyName("ece")] double Ece,
+ [property: JsonPropertyName("reliability_bins")] IReadOnlyList ReliabilityBins,
+ [property: JsonPropertyName("risk_coverage")] IReadOnlyList RiskCoverage);
+
+public static class CalibrationMetrics
+{
+ private static readonly double[] CoverageThresholds = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9, 0.95, 0.99];
+
+ public static CalibrationMetricsResult Measure(IReadOnlyList rows)
+ {
+ ArgumentNullException.ThrowIfNull(rows);
+ if (rows.Count == 0) throw new InvalidDataException("No labeled predictions.");
+ var bins = Enumerable.Range(0, 10).Select(_ => new List<(double Confidence, bool Correct)>()).ToArray();
+ var ranked = new List<(double Confidence, bool Correct)>(rows.Count);
+ var correctCount = 0;
+ var nll = 0.0;
+ var brier = 0.0;
+
+ foreach (var row in rows)
+ {
+ var normalized = Normalize(row);
+ var expectedIndex = IndexOfLabel(normalized.CandidateKeys, row.Label);
+ var predictedIndex = IndexOfMaximum(normalized.Probabilities);
+ var correct = predictedIndex == expectedIndex;
+ var topProbability = normalized.Probabilities[predictedIndex];
+ bins[Math.Min(9, (int)(topProbability * 10))].Add((topProbability, correct));
+ ranked.Add((Confidence(normalized.AnswerType, normalized.Probabilities), correct));
+ if (correct) correctCount++;
+ nll -= Math.Log(Math.Max(normalized.Probabilities[expectedIndex], 1e-9));
+ for (var index = 0; index < normalized.Probabilities.Count; index++)
+ {
+ var error = normalized.Probabilities[index] - (index == expectedIndex ? 1 : 0);
+ brier += error * error;
+ }
+ }
+
+ var reliability = new List();
+ var ece = 0.0;
+ for (var index = 0; index < bins.Length; index++)
+ {
+ var values = bins[index];
+ if (values.Count == 0) continue;
+ var meanConfidence = values.Average(value => value.Confidence);
+ var accuracy = values.Count(value => value.Correct) / (double)values.Count;
+ ece += values.Count / (double)rows.Count * Math.Abs(meanConfidence - accuracy);
+ reliability.Add(new ReliabilityBin(index / 10.0, (index + 1) / 10.0, values.Count, meanConfidence, accuracy));
+ }
+
+ var riskCoverage = CoverageThresholds.Select(threshold =>
+ {
+ var selected = ranked.Where(value => value.Confidence >= threshold).ToArray();
+ return new RiskCoveragePoint(threshold, selected.Length / (double)rows.Count,
+ selected.Length == 0 ? null : 1 - selected.Count(value => value.Correct) / (double)selected.Length);
+ }).ToArray();
+ return new CalibrationMetricsResult(rows.Count, correctCount / (double)rows.Count, nll / rows.Count,
+ brier / rows.Count, ece, reliability, riskCoverage);
+ }
+
+ internal static LabeledPrediction Normalize(LabeledPrediction row)
+ {
+ if (row.AnswerType is not ("choice" or "score" or "noul") || row.CandidateKeys.Count < 2 ||
+ row.CandidateKeys.Count != row.Probabilities.Count || row.CandidateKeys.Distinct(StringComparer.Ordinal).Count() != row.CandidateKeys.Count ||
+ !row.CandidateKeys.Contains(row.Label, StringComparer.Ordinal) ||
+ row.Probabilities.Any(probability => !double.IsFinite(probability) || probability is < 0 or > 1))
+ {
+ throw new InvalidDataException("Invalid labeled probability distribution.");
+ }
+ var sum = row.Probabilities.Sum();
+ if (Math.Abs(sum - 1) > 0.002) throw new InvalidDataException("Invalid labeled probability sum.");
+ return row with { Probabilities = row.Probabilities.Select(probability => probability / sum).ToArray() };
+ }
+
+ internal static double Confidence(string answerType, IReadOnlyList probabilities)
+ => answerType == "noul"
+ ? probabilities.Max()
+ : Math.Max(0, 1 + probabilities.Sum(probability => probability * Math.Log(Math.Max(probability, 1e-12))) / Math.Log(probabilities.Count));
+
+ internal static int IndexOfMaximum(IReadOnlyList values)
+ {
+ var index = 0;
+ for (var candidate = 1; candidate < values.Count; candidate++)
+ if (values[candidate] > values[index]) index = candidate;
+ return index;
+ }
+
+ internal static int IndexOfLabel(IReadOnlyList values, string label)
+ {
+ for (var index = 0; index < values.Count; index++)
+ if (string.Equals(values[index], label, StringComparison.Ordinal)) return index;
+ throw new InvalidDataException("Label is not present in the candidate distribution.");
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Evaluation/CaseEvaluator.cs b/tools/laya_service/Evaluation/CaseEvaluator.cs
new file mode 100644
index 00000000..0dc06e3d
--- /dev/null
+++ b/tools/laya_service/Evaluation/CaseEvaluator.cs
@@ -0,0 +1,222 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+using OpenClaw.LayaService.Inference;
+using OpenClaw.LayaService.Models;
+using OpenClaw.LayaService.Protocol;
+
+namespace OpenClaw.LayaService.Evaluation;
+
+public static class CaseEvaluator
+{
+ private const int MaximumResponseBytes = 262144;
+ private static readonly JsonSerializerOptions ObservationJson = new()
+ {
+ DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+ };
+
+ public static async Task EvaluateAsync(
+ string datasetPath,
+ Uri endpoint,
+ string outputPath,
+ HttpClient http,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(endpoint);
+ ArgumentNullException.ThrowIfNull(http);
+ ValidateEndpoint(endpoint);
+ var fullOutputPath = Path.GetFullPath(outputPath);
+ var outputDirectory = Path.GetDirectoryName(fullOutputPath) ?? throw new ArgumentException("Invalid output path.");
+ Directory.CreateDirectory(outputDirectory);
+ var temporaryPath = Path.Combine(outputDirectory, "." + Path.GetFileName(fullOutputPath) + "." + Guid.NewGuid().ToString("N") + ".tmp");
+ var caseIds = new HashSet(StringComparer.Ordinal);
+ var observationsWritten = 0;
+ try
+ {
+ await using (var output = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None,
+ 65536, FileOptions.Asynchronous | FileOptions.WriteThrough))
+ await using (var writer = new StreamWriter(output, new UTF8Encoding(false), leaveOpen: true))
+ using (var input = new StreamReader(datasetPath, Encoding.UTF8, detectEncodingFromByteOrderMarks: false))
+ {
+ while (await input.ReadLineAsync(cancellationToken) is { } line)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ if (string.IsNullOrWhiteSpace(line)) continue;
+ using var caseDocument = JsonDocument.Parse(line, new JsonDocumentOptions { MaxDepth = 64 });
+ RejectDuplicateProperties(caseDocument.RootElement);
+ var (caseId, labels, request, requestElement) = BuildRequest(caseDocument.RootElement);
+ if (caseId.Length == 0 || !caseIds.Add(caseId)) throw new InvalidDataException("Case IDs must be nonempty and unique.");
+ var response = await SendAsync(http, endpoint, request, cancellationToken);
+ var caseFingerprint = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(StrictJson.Canonicalize(requestElement))));
+ foreach (var answerProperty in response.RawAnswers.EnumerateObject())
+ {
+ var questionId = answerProperty.Name;
+ var answer = answerProperty.Value;
+ var label = NormalizeLabel(labels.GetProperty(questionId), answer, request.Questions.GetProperty(questionId));
+ var observation = new Observation(caseId, caseFingerprint, questionId, response.Model,
+ response.Checkpoint, response.SchemaHash, response.SdkVersion, answer.Clone(), label);
+ var observationJson = JsonSerializer.SerializeToElement(observation, ObservationJson);
+ await writer.WriteAsync(StrictJson.Canonicalize(observationJson));
+ await writer.WriteAsync("\n");
+ observationsWritten++;
+ }
+ }
+ await writer.FlushAsync(cancellationToken);
+ await output.FlushAsync(cancellationToken);
+ output.Flush(flushToDisk: true);
+ }
+
+ if (observationsWritten == 0) throw new InvalidDataException("Empty evaluation dataset.");
+ File.Move(temporaryPath, fullOutputPath, overwrite: true);
+ return observationsWritten;
+ }
+ finally
+ {
+ if (File.Exists(temporaryPath)) File.Delete(temporaryPath);
+ }
+ }
+
+ public static void ValidateEndpoint(Uri endpoint)
+ {
+ if (!endpoint.IsAbsoluteUri || endpoint.Scheme != Uri.UriSchemeHttp || endpoint.Host != "127.0.0.1" ||
+ endpoint.Port is < 1 or > 65535 || !string.IsNullOrEmpty(endpoint.UserInfo) ||
+ !string.IsNullOrEmpty(endpoint.Query) || !string.IsNullOrEmpty(endpoint.Fragment) ||
+ endpoint.AbsolutePath != "/v1/decisions")
+ {
+ throw new ArgumentException("Use the local http://127.0.0.1:PORT/v1/decisions endpoint.", nameof(endpoint));
+ }
+ }
+
+ private static (string CaseId, JsonElement Labels, DecisionWireRequest Request, JsonElement RequestElement) BuildRequest(JsonElement input)
+ {
+ if (input.ValueKind != JsonValueKind.Object || !input.TryGetProperty("case_id", out var caseIdElement) ||
+ caseIdElement.ValueKind != JsonValueKind.String || !input.TryGetProperty("labels", out var labels) || labels.ValueKind != JsonValueKind.Object)
+ {
+ throw new InvalidDataException("Invalid evaluation case.");
+ }
+
+ var requestNode = JsonNode.Parse(input.GetRawText())!.AsObject();
+ requestNode.Remove("case_id");
+ requestNode.Remove("labels");
+ if (!requestNode.ContainsKey("questions"))
+ {
+ using var rubric = typeof(CaseEvaluator).Assembly.GetManifestResourceStream("OpenClaw.LayaService.Rubric.openclaw-laya-tiers-v1.json") is { } stream
+ ? JsonDocument.Parse(stream)
+ : throw new InvalidDataException("Default rubric is missing.");
+ requestNode["rubric_version"] = rubric.RootElement.GetProperty("rubric_version").GetString();
+ requestNode["questions"] = JsonNode.Parse(rubric.RootElement.GetProperty("questions").GetRawText());
+ }
+ if (!requestNode.ContainsKey("model")) requestNode["model"] = "laya@" + ModelManifest.DefaultRevision;
+ using var requestDocument = JsonDocument.Parse(requestNode.ToJsonString());
+ var requestElement = requestDocument.RootElement.Clone();
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(StrictJson.Canonicalize(requestElement)));
+ RequestValidator.Validate(request, request.Model);
+ if (labels.GetPropertyCount() != request.Questions.GetPropertyCount() ||
+ request.Questions.EnumerateObject().Any(question => !labels.TryGetProperty(question.Name, out _)))
+ {
+ throw new InvalidDataException("Labels must cover every question exactly once.");
+ }
+ return (caseIdElement.GetString()!, labels.Clone(), request, requestElement);
+ }
+
+ private static async Task SendAsync(HttpClient http, Uri endpoint, DecisionWireRequest request, CancellationToken cancellationToken)
+ {
+ var payload = Encoding.UTF8.GetBytes(StrictJson.Canonicalize(JsonSerializer.SerializeToElement(request)));
+ using var message = new HttpRequestMessage(HttpMethod.Post, endpoint)
+ {
+ Content = new ByteArrayContent(payload)
+ };
+ message.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json");
+ using var response = await http.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (response.StatusCode is >= HttpStatusCode.MultipleChoices and < HttpStatusCode.BadRequest || !response.IsSuccessStatusCode ||
+ response.RequestMessage?.RequestUri != endpoint)
+ {
+ throw new InvalidDataException("Evaluation endpoint returned an unusable response.");
+ }
+
+ await using var body = await response.Content.ReadAsStreamAsync(cancellationToken);
+ var bytes = await ReadBoundedAsync(body, cancellationToken);
+ using var document = JsonDocument.Parse(bytes, new JsonDocumentOptions { MaxDepth = 64 });
+ RejectDuplicateProperties(document.RootElement);
+ var root = document.RootElement;
+ var metadata = root.GetProperty("metadata");
+ var model = root.GetProperty("model").GetString();
+ var checkpoint = metadata.GetProperty("checkpoint").GetString();
+ var schemaHash = metadata.GetProperty("schema_hash").GetString();
+ var sdkVersion = metadata.GetProperty("sdk_version").GetString();
+ if (model != request.Model || checkpoint is null || !ModelManifest.CheckpointNames.Contains(checkpoint) ||
+ metadata.GetProperty("revision").GetString() != request.Model[5..] ||
+ metadata.GetProperty("schema_hash").GetString() != StrictJson.SchemaHash(request.Questions) ||
+ metadata.GetProperty("rubric_version").GetString() != request.RubricVersion ||
+ sdkVersion != NLayaDecisionPredictor.SdkVersion || metadata.GetProperty("runtime").GetString() != "NLaya" ||
+ metadata.GetProperty("truncated").GetBoolean() || !root.TryGetProperty("raw_answers", out var rawAnswers) ||
+ rawAnswers.ValueKind != JsonValueKind.Object || rawAnswers.GetPropertyCount() != request.Questions.GetPropertyCount())
+ {
+ throw new InvalidDataException("Evaluation response metadata mismatch.");
+ }
+ foreach (var question in request.Questions.EnumerateObject())
+ {
+ if (!rawAnswers.TryGetProperty(question.Name, out var answer)) throw new InvalidDataException("Evaluation answer missing.");
+ _ = AnswerDistribution.Parse(answer, question.Value);
+ }
+ return new EvaluationResponse(model, checkpoint, schemaHash!, sdkVersion!, rawAnswers.Clone());
+ }
+
+ private static async Task ReadBoundedAsync(Stream stream, CancellationToken cancellationToken)
+ {
+ using var output = new MemoryStream();
+ var buffer = new byte[8192];
+ while (true)
+ {
+ var read = await stream.ReadAsync(buffer.AsMemory(), cancellationToken);
+ if (read == 0) return output.ToArray();
+ if (output.Length + read > MaximumResponseBytes) throw new InvalidDataException("Evaluation response too large.");
+ output.Write(buffer, 0, read);
+ }
+ }
+
+ private static string NormalizeLabel(JsonElement label, JsonElement answer, JsonElement question)
+ {
+ var type = question.GetProperty("type").GetString();
+ if (type == "noul")
+ {
+ if (label.ValueKind is not (JsonValueKind.True or JsonValueKind.False)) throw new InvalidDataException("Noul labels must be JSON booleans.");
+ var normalized = label.GetBoolean() ? "true" : "false";
+ if (!AnswerDistribution.Parse(answer, question).Keys.Contains(normalized, StringComparer.Ordinal))
+ throw new InvalidDataException("Label is outside the question options.");
+ return normalized;
+ }
+
+ var value = label.ValueKind switch
+ {
+ JsonValueKind.String => label.GetString(),
+ JsonValueKind.Number => label.GetRawText(),
+ _ => null
+ };
+ if (value is null || !AnswerDistribution.Parse(answer, question).Keys.Contains(value, StringComparer.Ordinal))
+ throw new InvalidDataException("Label is outside the question options.");
+ return value;
+ }
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name)) throw new InvalidDataException("Duplicate JSON property.");
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray()) RejectDuplicateProperties(item);
+ }
+ }
+
+ private sealed record EvaluationResponse(string Model, string Checkpoint, string SchemaHash, string SdkVersion, JsonElement RawAnswers);
+}
diff --git a/tools/laya_service/Evaluation/Observation.cs b/tools/laya_service/Evaluation/Observation.cs
new file mode 100644
index 00000000..74b66c91
--- /dev/null
+++ b/tools/laya_service/Evaluation/Observation.cs
@@ -0,0 +1,22 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace OpenClaw.LayaService.Evaluation;
+
+public sealed record Observation(
+ [property: JsonPropertyName("case_id")] string CaseId,
+ [property: JsonPropertyName("case_fingerprint")] string CaseFingerprint,
+ [property: JsonPropertyName("question_id")] string QuestionId,
+ [property: JsonPropertyName("model")] string Model,
+ [property: JsonPropertyName("checkpoint")] string Checkpoint,
+ [property: JsonPropertyName("schema_hash")] string SchemaHash,
+ [property: JsonPropertyName("sdk_version")] string RuntimeVersion,
+ [property: JsonPropertyName("answer")] JsonElement RawAnswer,
+ [property: JsonPropertyName("label")] string Label)
+{
+ [JsonPropertyName("runtime")]
+ public string Runtime { get; init; } = "NLaya";
+
+ [JsonPropertyName("source_calibration")]
+ public string SourceCalibration { get; init; } = "raw";
+}
\ No newline at end of file
diff --git a/tools/laya_service/Hosting/DecisionServer.cs b/tools/laya_service/Hosting/DecisionServer.cs
new file mode 100644
index 00000000..c0985e39
--- /dev/null
+++ b/tools/laya_service/Hosting/DecisionServer.cs
@@ -0,0 +1,181 @@
+using System.Net;
+using System.Text;
+using System.Text.Json;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using OpenClaw.LayaService.Protocol;
+
+namespace OpenClaw.LayaService.Hosting;
+
+public static class DecisionServer
+{
+ private const string DecisionsPath = "/v1/decisions";
+ private const string HealthPath = "/health";
+
+ public static WebApplication Build(IDecisionPredictor predictor, ServiceOptions options)
+ {
+ ArgumentNullException.ThrowIfNull(predictor);
+ ArgumentNullException.ThrowIfNull(options);
+ if (options.Port is < 0 or > 65535 || options.MaxConnections is < 1 or > 128 ||
+ options.MaxRequestBytes is < 1 or > 65536 || options.RequestTimeout <= TimeSpan.Zero)
+ {
+ throw new ArgumentOutOfRangeException(nameof(options));
+ }
+
+ var builder = WebApplication.CreateBuilder(new WebApplicationOptions
+ {
+ Args = [],
+ EnvironmentName = Environments.Production
+ });
+ builder.Logging.ClearProviders();
+ builder.WebHost.ConfigureKestrel(server =>
+ {
+ server.Limits.MaxConcurrentConnections = options.MaxConnections;
+ server.Limits.RequestHeadersTimeout = options.RequestTimeout;
+ server.Limits.KeepAliveTimeout = options.RequestTimeout;
+ server.Limits.MaxRequestBodySize = null;
+ server.Listen(IPAddress.Loopback, options.Port);
+ });
+
+ var app = builder.Build();
+ var inferenceGate = new SemaphoreSlim(1, 1);
+ app.Lifetime.ApplicationStopped.Register(inferenceGate.Dispose);
+ app.Run(context => HandleAsync(context, predictor, options, inferenceGate));
+ return app;
+ }
+
+ private static async Task HandleAsync(
+ HttpContext context,
+ IDecisionPredictor predictor,
+ ServiceOptions options,
+ SemaphoreSlim inferenceGate)
+ {
+ context.Response.Headers.CacheControl = "no-store";
+ if (!IsLocalRequest(context))
+ {
+ await WriteErrorAsync(context, StatusCodes.Status403Forbidden, "local_clients_only");
+ return;
+ }
+
+ if (HttpMethods.IsGet(context.Request.Method) && context.Request.Path == HealthPath)
+ {
+ try
+ {
+ await WriteJsonAsync(context, StatusCodes.Status200OK, predictor.GetHealth(), context.RequestAborted);
+ }
+ catch
+ {
+ await WriteErrorAsync(context, StatusCodes.Status503ServiceUnavailable, "health_unavailable");
+ }
+ return;
+ }
+
+ if (!HttpMethods.IsPost(context.Request.Method) || context.Request.Path != DecisionsPath)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status404NotFound, "not_found");
+ return;
+ }
+
+ if (context.Request.Headers.ContainsKey("Transfer-Encoding") ||
+ !string.Equals(context.Request.ContentType?.Split(';', 2)[0].Trim(), "application/json", StringComparison.OrdinalIgnoreCase) ||
+ context.Request.ContentLength is null)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status415UnsupportedMediaType, "json_content_length_required");
+ return;
+ }
+
+ var contentLength = context.Request.ContentLength.Value;
+ if (contentLength <= 0 || contentLength > options.MaxRequestBytes)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status413PayloadTooLarge, "request_size");
+ return;
+ }
+
+ if (!inferenceGate.Wait(0))
+ {
+ await WriteErrorAsync(context, StatusCodes.Status503ServiceUnavailable, "busy");
+ return;
+ }
+
+ try
+ {
+ using var deadline = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted);
+ deadline.CancelAfter(options.RequestTimeout);
+ var body = new byte[(int)contentLength];
+ await context.Request.Body.ReadExactlyAsync(body.AsMemory(), deadline.Token);
+
+ DecisionWireRequest request;
+ try
+ {
+ request = StrictJson.ParseRequest(body);
+ RequestValidator.Validate(request, predictor.Model);
+ }
+ catch (ProtocolRejectionException exception)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status422UnprocessableEntity, exception.ReasonCode);
+ return;
+ }
+
+ try
+ {
+ var result = await predictor.PredictAsync(request, deadline.Token);
+ await WriteJsonAsync(context, StatusCodes.Status200OK, result, deadline.Token);
+ }
+ catch (ProtocolRejectionException exception)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status422UnprocessableEntity, exception.ReasonCode);
+ }
+ catch
+ {
+ await WriteErrorAsync(context, StatusCodes.Status503ServiceUnavailable, "inference_failed");
+ }
+ }
+ catch (OperationCanceledException)
+ {
+ if (!context.Response.HasStarted)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status503ServiceUnavailable, "inference_failed");
+ }
+ }
+ catch
+ {
+ if (!context.Response.HasStarted)
+ {
+ await WriteErrorAsync(context, StatusCodes.Status503ServiceUnavailable, "inference_failed");
+ }
+ }
+ finally
+ {
+ inferenceGate.Release();
+ }
+ }
+
+ private static bool IsLocalRequest(HttpContext context)
+ {
+ if (context.Request.Headers.ContainsKey("Origin")) return false;
+ var expectedHost = $"127.0.0.1:{context.Connection.LocalPort}";
+ return string.Equals(context.Request.Host.Value, expectedHost, StringComparison.Ordinal);
+ }
+
+ private static async Task WriteErrorAsync(HttpContext context, int statusCode, string reasonCode)
+ {
+ var body = Encoding.UTF8.GetBytes("{\"error\":\"" + reasonCode + "\"}");
+ context.Response.StatusCode = statusCode;
+ context.Response.ContentType = "application/json";
+ context.Response.ContentLength = body.Length;
+ await context.Response.Body.WriteAsync(body, context.RequestAborted);
+ }
+
+ private static async Task WriteJsonAsync(HttpContext context, int statusCode, JsonElement value, CancellationToken cancellationToken)
+ {
+ var body = JsonSerializer.SerializeToUtf8Bytes(value);
+ context.Response.StatusCode = statusCode;
+ context.Response.ContentType = "application/json";
+ context.Response.ContentLength = body.Length;
+ await context.Response.Body.WriteAsync(body, cancellationToken);
+ }
+}
diff --git a/tools/laya_service/Hosting/ServiceContracts.cs b/tools/laya_service/Hosting/ServiceContracts.cs
new file mode 100644
index 00000000..b73b87f3
--- /dev/null
+++ b/tools/laya_service/Hosting/ServiceContracts.cs
@@ -0,0 +1,15 @@
+using System.Text.Json;
+using OpenClaw.LayaService.Protocol;
+
+namespace OpenClaw.LayaService.Hosting;
+
+public sealed record ServiceOptions(int Port, int MaxConnections, int MaxRequestBytes, TimeSpan RequestTimeout);
+
+public interface IDecisionPredictor
+{
+ string Model { get; }
+
+ JsonElement GetHealth();
+
+ Task PredictAsync(DecisionWireRequest request, CancellationToken cancellationToken);
+}
\ No newline at end of file
diff --git a/tools/laya_service/Inference/CalibrationStore.cs b/tools/laya_service/Inference/CalibrationStore.cs
new file mode 100644
index 00000000..62b7db96
--- /dev/null
+++ b/tools/laya_service/Inference/CalibrationStore.cs
@@ -0,0 +1,202 @@
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using NLaya;
+using OpenClaw.LayaService.Models;
+using OpenClaw.LayaService.Protocol;
+
+namespace OpenClaw.LayaService.Inference;
+
+public sealed class CalibrationStore
+{
+ private readonly string? _schemaHash;
+ private readonly IReadOnlyDictionary> _temperatures;
+
+ private CalibrationStore(string identifier, string? schemaHash, IReadOnlyDictionary> temperatures)
+ {
+ Identifier = identifier;
+ _schemaHash = schemaHash;
+ _temperatures = temperatures;
+ }
+
+ public string Identifier { get; }
+
+ public static CalibrationStore Load(string? path, string model)
+ {
+ if (string.IsNullOrWhiteSpace(path))
+ {
+ return new CalibrationStore("uncalibrated", null, new Dictionary>(StringComparer.Ordinal));
+ }
+
+ var payload = File.ReadAllBytes(path);
+ using var document = JsonDocument.Parse(payload, new JsonDocumentOptions { MaxDepth = 32 });
+ RejectDuplicateProperties(document.RootElement);
+ var root = document.RootElement;
+ if (root.ValueKind != JsonValueKind.Object ||
+ !root.TryGetProperty("version", out var version) || version.ValueKind != JsonValueKind.Number || version.GetInt32() != 2 ||
+ !root.TryGetProperty("model", out var artifactModel) || artifactModel.ValueKind != JsonValueKind.String || artifactModel.GetString() != model)
+ {
+ throw new InvalidDataException("Invalid calibration artifact or model identity.");
+ }
+
+ var schema = root.TryGetProperty("schema_hash", out var schemaValue) ? schemaValue : default;
+ var sdk = root.TryGetProperty("sdk_version", out var sdkValue) ? sdkValue : default;
+ var runtime = root.TryGetProperty("runtime", out var runtimeValue) ? runtimeValue : default;
+ var temperaturesElement = root.TryGetProperty("temperatures", out var temperaturesValue) ? temperaturesValue : default;
+ var validation = root.TryGetProperty("validation", out var validationValue) ? validationValue : default;
+ if (schema.ValueKind != JsonValueKind.String || !ModelManifest.IsHash(schema.GetString()) ||
+ sdk.ValueKind != JsonValueKind.String || sdk.GetString() != NLayaDecisionPredictor.SdkVersion ||
+ runtime.ValueKind != JsonValueKind.String || runtime.GetString() != "NLaya" ||
+ temperaturesElement.ValueKind != JsonValueKind.Object || temperaturesElement.GetPropertyCount() == 0 ||
+ validation.ValueKind != JsonValueKind.Object)
+ {
+ throw new InvalidDataException("Invalid calibration artifact or model identity.");
+ }
+
+ var temperatures = new Dictionary>(StringComparer.Ordinal);
+ foreach (var checkpoint in temperaturesElement.EnumerateObject())
+ {
+ if (!ModelManifest.CheckpointNames.Contains(checkpoint.Name) || checkpoint.Value.ValueKind != JsonValueKind.Object ||
+ checkpoint.Value.GetPropertyCount() == 0)
+ {
+ throw new InvalidDataException("Invalid calibration checkpoint.");
+ }
+
+ var values = new Dictionary(StringComparer.Ordinal);
+ foreach (var bucket in checkpoint.Value.EnumerateObject())
+ {
+ var parts = bucket.Name.Split(':');
+ if (parts.Length != 2 || parts[0] is not ("choice" or "score" or "noul") ||
+ !int.TryParse(parts[1], out var count) || count < 2 || bucket.Value.ValueKind != JsonValueKind.Number ||
+ !bucket.Value.TryGetDouble(out var temperature) || !double.IsFinite(temperature) || temperature is < 0.1 or > 10)
+ {
+ throw new InvalidDataException("Invalid calibration temperature.");
+ }
+ values.Add(bucket.Name, temperature);
+ }
+ temperatures.Add(checkpoint.Name, values);
+ }
+
+ return new CalibrationStore(Convert.ToHexStringLower(SHA256.HashData(payload)), schema.GetString(), temperatures);
+ }
+
+ public void ValidateRequest(DecisionWireRequest request, string checkpoint, Questions questions)
+ {
+ if (_schemaHash is null) return;
+ if (!string.Equals(_schemaHash, StrictJson.SchemaHash(request.Questions), StringComparison.Ordinal))
+ {
+ throw new ProtocolRejectionException("calibration_schema_mismatch");
+ }
+ if (!_temperatures.TryGetValue(checkpoint, out var checkpointTemperatures) ||
+ questions.Values.Any(question => !checkpointTemperatures.ContainsKey(Bucket(question.TypeName, question.OptionCount))))
+ {
+ throw new ProtocolRejectionException("calibration_bucket_missing");
+ }
+ }
+
+ public JsonElement Apply(DecisionWireRequest request, string checkpoint, JsonElement answers)
+ {
+ if (_schemaHash is null) return answers.Clone();
+ if (!string.Equals(_schemaHash, StrictJson.SchemaHash(request.Questions), StringComparison.Ordinal) ||
+ !_temperatures.TryGetValue(checkpoint, out var checkpointTemperatures))
+ {
+ throw new ProtocolRejectionException("calibration_schema_mismatch");
+ }
+
+ var questions = Questions.Parse(request.Questions.GetRawText());
+ var transformed = JsonNode.Parse(answers.GetRawText())!.AsObject();
+ foreach (var (questionId, question) in questions)
+ {
+ if (!checkpointTemperatures.TryGetValue(Bucket(question.TypeName, question.OptionCount), out var temperature) ||
+ !transformed.TryGetPropertyValue(questionId, out var answerNode) || answerNode is not JsonObject answer)
+ {
+ throw new ProtocolRejectionException("calibration_bucket_missing");
+ }
+ TransformAnswer(answer, question.TypeName, question, temperature);
+ }
+ using var document = JsonDocument.Parse(transformed.ToJsonString());
+ return document.RootElement.Clone();
+ }
+
+ private static void TransformAnswer(JsonObject answer, string type, Question question, double temperature)
+ {
+ var labels = type switch
+ {
+ "choice" => question.Options!.Select(option => option.Key).ToArray(),
+ "score" => Enumerable.Range(0, question.Levels!.Count).Select(index => index.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToArray(),
+ "noul" => ["false", "true"],
+ _ => throw new InvalidDataException("Invalid calibration answer type.")
+ };
+ double[] probabilities;
+ if (type == "noul")
+ {
+ if (!answer.TryGetPropertyValue("noul", out var positive) || positive is not JsonValue value || !value.TryGetValue(out var probability) || probability is < 0 or > 1)
+ throw new InvalidDataException("Invalid noul probabilities.");
+ probabilities = [1 - probability, probability];
+ }
+ else
+ {
+ if (!answer.TryGetPropertyValue("probabilities", out var probabilityNode) || probabilityNode is not JsonObject distribution ||
+ distribution.Count != labels.Length || labels.Any(label => !distribution.ContainsKey(label)))
+ throw new InvalidDataException("Invalid answer probabilities.");
+ probabilities = labels.Select(label => distribution[label]!.GetValue()).ToArray();
+ if (probabilities.Any(value => !double.IsFinite(value) || value is < 0 or > 1))
+ throw new InvalidDataException("Invalid answer probabilities.");
+ var total = probabilities.Sum();
+ if (Math.Abs(total - 1) > 0.002)
+ throw new InvalidDataException("Invalid answer probabilities.");
+ for (var index = 0; index < probabilities.Length; index++) probabilities[index] /= total;
+ }
+
+ var scaled = Scale(probabilities, temperature);
+ var maxIndex = Array.IndexOf(scaled, scaled.Max());
+ if (type == "noul")
+ {
+ answer["noul"] = scaled[1];
+ answer["value"] = scaled[1] >= 0.5;
+ answer["answer_confidence"] = Math.Max(scaled[0], scaled[1]);
+ }
+ else
+ {
+ var outputProbabilities = new JsonObject();
+ for (var index = 0; index < labels.Length; index++) outputProbabilities[labels[index]] = scaled[index];
+ answer["probabilities"] = outputProbabilities;
+ answer["answer_confidence"] = scaled[maxIndex];
+ if (type == "choice") answer["choice"] = labels[maxIndex];
+ else answer["score"] = scaled.Select((probability, index) => probability * index).Sum();
+ }
+
+ answer["confidence"] = type == "noul" ? Math.Max(scaled[0], scaled[1]) : EntropyConfidence(scaled);
+ }
+
+ private static double[] Scale(IReadOnlyList probabilities, double temperature)
+ {
+ var logits = probabilities.Select(probability => Math.Log(Math.Max(probability, 1e-9)) / temperature).ToArray();
+ var peak = logits.Max();
+ var exponents = logits.Select(value => Math.Exp(value - peak)).ToArray();
+ var total = exponents.Sum();
+ return exponents.Select(value => value / total).ToArray();
+ }
+
+ private static double EntropyConfidence(IReadOnlyList probabilities)
+ => Math.Max(0, 1 + probabilities.Sum(probability => probability * Math.Log(Math.Max(probability, 1e-12))) / Math.Log(probabilities.Count));
+
+ private static string Bucket(string type, int options) => type + ":" + options.ToString(System.Globalization.CultureInfo.InvariantCulture);
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name)) throw new InvalidDataException("Duplicate calibration property.");
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray()) RejectDuplicateProperties(item);
+ }
+ }
+}
diff --git a/tools/laya_service/Inference/NLayaDecisionPredictor.cs b/tools/laya_service/Inference/NLayaDecisionPredictor.cs
new file mode 100644
index 00000000..ccb5de44
--- /dev/null
+++ b/tools/laya_service/Inference/NLayaDecisionPredictor.cs
@@ -0,0 +1,307 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using NLaya;
+using NLaya.Lang;
+using NLaya.Routing;
+using NLaya.TorchSharp;
+using OpenClaw.LayaService.Hosting;
+using OpenClaw.LayaService.Models;
+using OpenClaw.LayaService.Protocol;
+using TorchSharp;
+
+namespace OpenClaw.LayaService.Inference;
+
+public sealed class NLayaDecisionPredictor : IDecisionPredictor, IAsyncDisposable
+{
+ public const string SdkVersion = "1.0.0";
+ private readonly Router _router;
+ private readonly IReadOnlyDictionary _agents;
+ private readonly CalibrationStore _calibration;
+ private readonly string _configuredCheckpoint;
+ private readonly string _device;
+ private readonly string _revision;
+ private readonly IReadOnlyDictionary _backendNames;
+
+ private NLayaDecisionPredictor(
+ string model,
+ string configuredCheckpoint,
+ string device,
+ Router router,
+ IReadOnlyDictionary agents,
+ CalibrationStore calibration,
+ string revision)
+ {
+ Model = model;
+ _configuredCheckpoint = configuredCheckpoint;
+ _device = device;
+ _router = router;
+ _agents = agents;
+ _calibration = calibration;
+ _revision = revision;
+ _backendNames = agents.ToDictionary(pair => pair.Key, pair => pair.Value.Backend.Name, StringComparer.Ordinal);
+ }
+
+ public string Model { get; }
+
+ public static async Task LoadAsync(
+ VerifiedManifest manifest,
+ ServeOptions options,
+ CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(manifest);
+ ArgumentNullException.ThrowIfNull(options);
+ if (!ModelManifest.IsRevision(manifest.Revision) || options.Threads < 1 || options.Port is < 0 or > 65535 ||
+ options.Device is not ("cpu" or "cuda" or "mps"))
+ {
+ throw new ArgumentException("Invalid serving options.");
+ }
+
+ var configuredCheckpoint = NormalizeConfiguredCheckpoint(options.Checkpoint);
+ if (configuredCheckpoint != "auto" && !manifest.Checkpoints.ContainsKey(configuredCheckpoint))
+ {
+ throw new InvalidOperationException("checkpoint_not_installed");
+ }
+
+ var model = "laya@" + manifest.Revision;
+ var calibration = CalibrationStore.Load(options.CalibrationPath, model);
+ var router = new Router(new RouterOptions { AutoTaskDetection = false });
+ var names = configuredCheckpoint == "auto"
+ ? manifest.Checkpoints.Keys.Where(name => name is "english" or "multilingual").ToArray()
+ : [configuredCheckpoint];
+ if (names.Length == 0)
+ {
+ router.Dispose();
+ throw new InvalidOperationException("checkpoint_not_installed");
+ }
+
+ var agents = new Dictionary(StringComparer.Ordinal);
+ try
+ {
+ torch.set_num_threads(options.Threads);
+ foreach (var name in names)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var checkpoint = manifest.Checkpoints[name];
+ var agent = await Laya.LoadAsync(checkpoint.AbsolutePath,
+ configure => configure.UseTorchSharp(options.Device), cancellationToken);
+ agents.Add(name, agent);
+ EnsureDevice(agent.Backend.Name, options.Device);
+ agent.Warmup();
+ }
+
+ return new NLayaDecisionPredictor(model, configuredCheckpoint, options.Device, router, agents, calibration, manifest.Revision);
+ }
+ catch
+ {
+ foreach (var agent in agents.Values) await agent.DisposeAsync();
+ router.Dispose();
+ throw;
+ }
+ }
+
+ public JsonElement GetHealth()
+ {
+ var payload = JsonSerializer.SerializeToElement(new
+ {
+ ready = true,
+ model = Model,
+ calibration_id = _calibration.Identifier,
+ sdk_version = SdkVersion,
+ runtime = "NLaya",
+ checkpoints = _backendNames
+ });
+ return payload.Clone();
+ }
+
+ public async Task PredictAsync(DecisionWireRequest request, CancellationToken cancellationToken)
+ {
+ var state = LayaState.ParseJson(request.State.GetRawText());
+ var questions = Questions.Parse(request.Questions.GetRawText());
+ var checkpoint = ResolveCheckpoint(request, _configuredCheckpoint, _agents.Keys.ToHashSet(StringComparer.Ordinal), _router, state, questions);
+ var agent = _agents[checkpoint];
+ EnsureWithinTokenBudget(agent, state, questions);
+ _calibration.ValidateRequest(request, checkpoint, questions);
+
+ var prediction = await agent.PredictAsync(state, questions,
+ new PredictOptions { Lang = request.Language }, cancellationToken);
+ using var predictionDocument = JsonDocument.Parse(prediction.ToJsonString());
+ var result = predictionDocument.RootElement.Clone();
+ var rawAnswers = result.GetProperty("answers");
+ var calibratedAnswers = _calibration.Apply(request, checkpoint, rawAnswers);
+ var response = MapPrediction(request, result, checkpoint, GetDevice(agent.Backend.Name), _calibration.Identifier, calibratedAnswers);
+ return response;
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ _router.Dispose();
+ foreach (var agent in _agents.Values)
+ {
+ await agent.DisposeAsync();
+ }
+ }
+
+ public static string ResolveCheckpoint(DecisionWireRequest request, string configuredCheckpoint, IReadOnlySet installedCheckpoints)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ ArgumentNullException.ThrowIfNull(installedCheckpoints);
+ var router = new Router(new RouterOptions { AutoTaskDetection = false });
+ try
+ {
+ var state = LayaState.ParseJson(request.State.GetRawText());
+ var questions = Questions.Parse(request.Questions.GetRawText());
+ return ResolveCheckpoint(request, NormalizeConfiguredCheckpoint(configuredCheckpoint), installedCheckpoints, router, state, questions);
+ }
+ finally
+ {
+ router.Dispose();
+ }
+ }
+
+ public static JsonElement MapPrediction(
+ DecisionWireRequest request,
+ JsonElement prediction,
+ string checkpoint,
+ string device,
+ string calibrationId,
+ JsonElement? calibratedAnswers = null)
+ {
+ if (!ModelManifest.CheckpointNames.Contains(checkpoint) || prediction.ValueKind != JsonValueKind.Object ||
+ !prediction.TryGetProperty("answers", out var rawAnswers) || rawAnswers.ValueKind != JsonValueKind.Object ||
+ !prediction.TryGetProperty("usage", out var usage) || usage.ValueKind != JsonValueKind.Object)
+ {
+ throw new InvalidDataException("Invalid NLaya prediction.");
+ }
+
+ var result = new JsonObject
+ {
+ ["model"] = request.Model,
+ ["answers"] = JsonNode.Parse((calibratedAnswers ?? rawAnswers).GetRawText()),
+ ["raw_answers"] = JsonNode.Parse(rawAnswers.GetRawText()),
+ ["usage"] = JsonNode.Parse(usage.GetRawText()),
+ ["metadata"] = new JsonObject
+ {
+ ["checkpoint"] = checkpoint,
+ ["revision"] = request.Model[5..],
+ ["calibration_id"] = calibrationId,
+ ["schema_hash"] = StrictJson.SchemaHash(request.Questions),
+ ["rubric_version"] = request.RubricVersion,
+ ["device"] = device,
+ ["sdk_version"] = SdkVersion,
+ ["runtime"] = "NLaya",
+ ["truncated"] = false
+ }
+ };
+ using var resultDocument = JsonDocument.Parse(result.ToJsonString());
+ return resultDocument.RootElement.Clone();
+ }
+
+ public static void EnsureDevice(string backendName, string requestedDevice)
+ {
+ if (!backendName.EndsWith(":" + requestedDevice, StringComparison.OrdinalIgnoreCase))
+ {
+ throw new InvalidOperationException("requested_device_unavailable");
+ }
+ }
+
+ private static string ResolveCheckpoint(
+ DecisionWireRequest request,
+ string configuredCheckpoint,
+ IReadOnlySet installedCheckpoints,
+ Router router,
+ LayaState state,
+ Questions questions)
+ {
+ var routeOptions = new RouteOptions
+ {
+ Lang = request.Language
+ };
+
+ var detected = Checkpoints.Name(router.Route(state, questions, routeOptions).Checkpoint);
+ if (IsEnglishLanguage(request.Language) && !LanguageDetector.IsEnglish(state))
+ {
+ throw new ProtocolRejectionException("language_checkpoint_conflict");
+ }
+
+ var selected = configuredCheckpoint == "auto" ? detected : configuredCheckpoint;
+ if (configuredCheckpoint == "english" && detected != "english" ||
+ configuredCheckpoint == "typed-decisions" && IsEnglishLanguage(request.Language) && !LanguageDetector.IsEnglish(state))
+ {
+ throw new ProtocolRejectionException("language_checkpoint_conflict");
+ }
+ if (!installedCheckpoints.Contains(selected)) throw new ProtocolRejectionException("checkpoint_not_installed");
+ return selected;
+ }
+
+ private static string NormalizeConfiguredCheckpoint(string? checkpoint)
+ {
+ if (string.Equals(checkpoint, "auto", StringComparison.Ordinal)) return "auto";
+ if (checkpoint is not null && Checkpoints.TryParse(checkpoint, out var parsed) && parsed is not null)
+ {
+ return Checkpoints.Name(parsed.Value);
+ }
+ throw new InvalidOperationException("unknown_checkpoint");
+ }
+
+ private static void EnsureWithinTokenBudget(LayaAgent agent, LayaState state, Questions questions)
+ {
+ var maskToken = agent.Tokenizer.MaskToken;
+ var serializedState = state.Serialize();
+ EnsureReservedTokenAbsent(serializedState, maskToken);
+ var stateTokenCount = agent.Tokenizer.Encode(serializedState).Length;
+ foreach (var question in questions.Values)
+ {
+ var instruction = question.InstructionText;
+ EnsureReservedTokenAbsent(instruction, maskToken);
+ var options = question.RenderOptions();
+ var optionLengths = options.Select(option =>
+ {
+ EnsureReservedTokenAbsent(option, maskToken);
+ return 1 + agent.Tokenizer.Encode(" " + option).Length;
+ }).ToArray();
+ EnsureOptionTokenBudget(optionLengths);
+ var headTokenCount = agent.Tokenizer.Encode(question.TypeName + " question: " + instruction).Length;
+ EnsureQuestionHeadBudget(headTokenCount, optionLengths, agent.Config.HeadMaxLen);
+ EnsureTokenBudget(4 + headTokenCount + optionLengths.Sum() + stateTokenCount, agent.Config.MaxLen);
+ }
+ }
+
+ public static void EnsureReservedTokenAbsent(string text, string maskToken)
+ {
+ if (text.Contains(maskToken, StringComparison.Ordinal))
+ {
+ throw new ProtocolRejectionException("reserved_token_in_input");
+ }
+ }
+
+ public static void EnsureOptionTokenBudget(IReadOnlyList optionLengths)
+ {
+ if (optionLengths.Any(length => length > 49))
+ {
+ throw new ProtocolRejectionException("option_too_long");
+ }
+ }
+
+ public static void EnsureQuestionHeadBudget(int headTokenCount, IReadOnlyList optionLengths, int headMaximumLength)
+ {
+ var available = headMaximumLength - optionLengths.Sum();
+ if (available < 16 || headTokenCount > Math.Max(8, available))
+ {
+ throw new ProtocolRejectionException("question_too_long");
+ }
+ }
+
+ public static void EnsureTokenBudget(int sequenceLength, int maximumLength)
+ {
+ if (sequenceLength > maximumLength)
+ {
+ throw new ProtocolRejectionException("input_exceeds_token_budget");
+ }
+ }
+
+ private static string GetDevice(string backendName)
+ => backendName[(backendName.LastIndexOf(':') + 1)..].ToLowerInvariant();
+
+ private static bool IsEnglishLanguage(string? language)
+ => language is not null && language.Split(['-', '_', '.'], 2)[0].Equals("en", StringComparison.OrdinalIgnoreCase);
+}
diff --git a/tools/laya_service/Inference/ServeOptions.cs b/tools/laya_service/Inference/ServeOptions.cs
new file mode 100644
index 00000000..865ec192
--- /dev/null
+++ b/tools/laya_service/Inference/ServeOptions.cs
@@ -0,0 +1,9 @@
+namespace OpenClaw.LayaService.Inference;
+
+public sealed record ServeOptions(
+ string ManifestPath,
+ string? CalibrationPath,
+ int Port,
+ string Device,
+ string Checkpoint,
+ int Threads);
\ No newline at end of file
diff --git a/tools/laya_service/LayaService.csproj b/tools/laya_service/LayaService.csproj
new file mode 100644
index 00000000..a7f68224
--- /dev/null
+++ b/tools/laya_service/LayaService.csproj
@@ -0,0 +1,20 @@
+
+
+ Exe
+ OpenClaw.LayaService
+ OpenClaw.LayaService
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tools/laya_service/Models/HuggingFaceDownloader.cs b/tools/laya_service/Models/HuggingFaceDownloader.cs
new file mode 100644
index 00000000..8fcd507f
--- /dev/null
+++ b/tools/laya_service/Models/HuggingFaceDownloader.cs
@@ -0,0 +1,302 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Net;
+
+namespace OpenClaw.LayaService.Models;
+
+public sealed record DownloadOptions(string Destination, string Revision, IReadOnlyList Checkpoints);
+
+public static class HuggingFaceDownloader
+{
+ private const string Repository = "convaiinnovations/laya";
+ private const string NoticeFile = "THIRD_PARTY_NOTICES.md";
+ private const string LicenseFile = "licenses/laya-APACHE-2.0.txt";
+
+ public static async Task DownloadAsync(DownloadOptions options, HttpClient http, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(options);
+ ArgumentNullException.ThrowIfNull(http);
+ if (!ModelManifest.IsRevision(options.Revision)) throw new ArgumentException("Revision must be a 40-character lowercase commit id.");
+ if (options.Checkpoints is null || options.Checkpoints.Count == 0 ||
+ options.Checkpoints.Any(checkpoint => !ModelManifest.CheckpointNames.Contains(checkpoint)) ||
+ options.Checkpoints.Distinct(StringComparer.Ordinal).Count() != options.Checkpoints.Count)
+ {
+ throw new ArgumentException("At least one unique supported checkpoint is required.");
+ }
+
+ var destination = Path.GetFullPath(options.Destination);
+ Directory.CreateDirectory(destination);
+ if ((new DirectoryInfo(destination).Attributes & FileAttributes.ReparsePoint) != 0)
+ {
+ throw new InvalidDataException("Model destination cannot be a symbolic link.");
+ }
+ var revisionDirectory = Path.Combine(destination, options.Revision);
+ Directory.CreateDirectory(revisionDirectory);
+ ModelManifest.RejectReparsePoints(destination, revisionDirectory);
+ var manifestPath = Path.Combine(revisionDirectory, "manifest.json");
+ var manifest = LoadOrCreateManifest(manifestPath, options.Revision);
+ var checkpointEntries = new Dictionary(manifest.Checkpoints, StringComparer.Ordinal);
+
+ foreach (var checkpoint in options.Checkpoints)
+ {
+ cancellationToken.ThrowIfCancellationRequested();
+ var checkpointPrefix = checkpoint == "english" ? string.Empty : checkpoint + "/";
+ var checkpointDirectory = checkpoint == "english"
+ ? Path.Combine(revisionDirectory, "hub")
+ : Path.Combine(revisionDirectory, "hub", checkpoint);
+ Directory.CreateDirectory(checkpointDirectory);
+ ModelManifest.RejectReparsePoints(revisionDirectory, checkpointDirectory);
+ var fileHashes = new Dictionary(StringComparer.Ordinal);
+
+ foreach (var file in ModelManifest.ModelFiles)
+ {
+ var target = Path.Combine(checkpointDirectory, file.Replace('/', Path.DirectorySeparatorChar));
+ Directory.CreateDirectory(Path.GetDirectoryName(target)!);
+ ModelManifest.RejectReparsePoints(revisionDirectory, Path.GetDirectoryName(target)!);
+ var sourceFile = checkpointPrefix + file;
+ var requestUri = new Uri($"https://huggingface.co/{Repository}/resolve/{options.Revision}/{sourceFile}");
+ await DownloadFileAsync(http, requestUri, target, checkpoint, file, options.Revision, cancellationToken);
+ fileHashes.Add(file, await HashFileAsync(target, cancellationToken));
+ }
+
+ checkpointEntries[checkpoint] = new CheckpointManifestEntry
+ {
+ Path = Path.GetRelativePath(revisionDirectory, checkpointDirectory).Replace(Path.DirectorySeparatorChar, '/'),
+ Sha256 = fileHashes
+ };
+ }
+
+ var updatedManifest = new ModelManifestDocument
+ {
+ Version = 1,
+ Revision = options.Revision,
+ Checkpoints = checkpointEntries
+ };
+ var temporaryManifest = manifestPath + "." + Guid.NewGuid().ToString("N") + ".tmp";
+ try
+ {
+ await File.WriteAllBytesAsync(temporaryManifest, JsonSerializer.SerializeToUtf8Bytes(updatedManifest, JsonOptions), cancellationToken);
+ _ = ModelManifest.LoadAndVerify(temporaryManifest, options.Revision);
+ File.Move(temporaryManifest, manifestPath, overwrite: true);
+ }
+ finally
+ {
+ if (File.Exists(temporaryManifest)) File.Delete(temporaryManifest);
+ }
+
+ CopyProvenanceFile(NoticeFile, Path.Combine(revisionDirectory, "LAYA-NOTICE.md"));
+ CopyProvenanceFile(LicenseFile, Path.Combine(revisionDirectory, "LAYA-LICENSE.txt"));
+ return manifestPath;
+ }
+
+ private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true };
+
+ private static ModelManifestDocument LoadOrCreateManifest(string path, string revision)
+ {
+ if (!File.Exists(path))
+ {
+ return new ModelManifestDocument { Version = 1, Revision = revision, Checkpoints = new(StringComparer.Ordinal) };
+ }
+
+ _ = ModelManifest.LoadAndVerify(path, revision);
+ return JsonSerializer.Deserialize(File.ReadAllBytes(path))
+ ?? throw new InvalidDataException("Existing model manifest is invalid.");
+ }
+
+ private static async Task DownloadFileAsync(
+ HttpClient http,
+ Uri requestUri,
+ string target,
+ string checkpoint,
+ string file,
+ string revision,
+ CancellationToken cancellationToken)
+ {
+ using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+ using var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (response.RequestMessage?.RequestUri != requestUri)
+ {
+ throw new HttpRequestException("Model download response did not match the requested URI.");
+ }
+
+ if (response.StatusCode == HttpStatusCode.Found)
+ {
+ if (!IsAllowedCdnRedirect(response.Headers.Location))
+ {
+ throw new HttpRequestException("Model download failed or was redirected.");
+ }
+
+ var redirectUri = response.Headers.Location!;
+ using var redirectRequest = new HttpRequestMessage(HttpMethod.Get, redirectUri);
+ using var redirectedResponse = await http.SendAsync(redirectRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ await StoreResponseAsync(redirectedResponse, redirectUri, target, checkpoint, file, revision, cancellationToken);
+ return;
+ }
+
+ if (response.StatusCode == HttpStatusCode.TemporaryRedirect)
+ {
+ var cacheUri = GetAllowedResolveCacheRedirect(response.Headers.Location, requestUri, checkpoint, file, revision);
+ if (cacheUri is null)
+ {
+ throw new HttpRequestException("Model download failed or was redirected.");
+ }
+
+ using var cacheRequest = new HttpRequestMessage(HttpMethod.Get, cacheUri);
+ using var cacheResponse = await http.SendAsync(cacheRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ if (cacheResponse.RequestMessage?.RequestUri != cacheUri)
+ {
+ throw new HttpRequestException("Model cache response did not match the requested URI.");
+ }
+
+ if (cacheResponse.StatusCode == HttpStatusCode.Found)
+ {
+ if (!IsAllowedCdnRedirect(cacheResponse.Headers.Location))
+ {
+ throw new HttpRequestException("Model download failed or was redirected.");
+ }
+
+ var cdnUri = cacheResponse.Headers.Location!;
+ using var cdnRequest = new HttpRequestMessage(HttpMethod.Get, cdnUri);
+ using var cdnResponse = await http.SendAsync(cdnRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+ await StoreResponseAsync(cdnResponse, cdnUri, target, checkpoint, file, revision, cancellationToken);
+ return;
+ }
+
+ await StoreResponseAsync(cacheResponse, cacheUri, target, checkpoint, file, revision, cancellationToken);
+ return;
+ }
+
+ await StoreResponseAsync(response, requestUri, target, checkpoint, file, revision, cancellationToken);
+ }
+
+ private static bool IsAllowedCdnRedirect(Uri? redirectUri)
+ {
+ if (redirectUri is not { IsAbsoluteUri: true } || redirectUri.Scheme != Uri.UriSchemeHttps ||
+ redirectUri.Port != 443 || redirectUri.UserInfo.Length != 0 || redirectUri.Fragment.Length != 0)
+ {
+ return false;
+ }
+
+ return redirectUri.IdnHost.EndsWith(".cdn.hf.co", StringComparison.OrdinalIgnoreCase);
+ }
+
+ private static Uri? GetAllowedResolveCacheRedirect(Uri? location, Uri requestUri, string checkpoint, string file, string revision)
+ {
+ if (location is null) return null;
+ var redirectUri = location.IsAbsoluteUri ? location : new Uri(requestUri, location);
+ var sourceFile = (checkpoint == "english" ? string.Empty : checkpoint + "/") + file;
+ var expectedPath = $"/api/resolve-cache/models/{Repository}/{revision}/{Uri.EscapeDataString(sourceFile)}";
+ return redirectUri.Scheme == Uri.UriSchemeHttps && redirectUri.IdnHost.Equals("huggingface.co", StringComparison.OrdinalIgnoreCase) &&
+ redirectUri.Port == 443 && redirectUri.UserInfo.Length == 0 && redirectUri.Fragment.Length == 0 &&
+ redirectUri.AbsolutePath.Equals(expectedPath, StringComparison.Ordinal) && redirectUri.Query.Length > 0
+ ? redirectUri
+ : null;
+ }
+
+ private static async Task StoreResponseAsync(
+ HttpResponseMessage response,
+ Uri expectedUri,
+ string target,
+ string checkpoint,
+ string file,
+ string revision,
+ CancellationToken cancellationToken)
+ {
+ if (response.StatusCode != HttpStatusCode.OK || response.Headers.Location is not null ||
+ response.RequestMessage?.RequestUri != expectedUri)
+ {
+ throw new HttpRequestException("Model download failed or was redirected.");
+ }
+
+ var temporary = target + "." + Guid.NewGuid().ToString("N") + ".tmp";
+ try
+ {
+ await using (var output = new FileStream(temporary, new FileStreamOptions
+ {
+ Mode = FileMode.CreateNew,
+ Access = FileAccess.Write,
+ Share = FileShare.None,
+ Options = FileOptions.Asynchronous | FileOptions.WriteThrough
+ }))
+ await using (var input = await response.Content.ReadAsStreamAsync(cancellationToken))
+ {
+ await input.CopyToAsync(output, cancellationToken);
+ await output.FlushAsync(cancellationToken);
+ }
+
+ var upstreamHash = await HashFileAsync(temporary, cancellationToken);
+ var expectedHash = ModelManifest.GetPinnedHash(revision, checkpoint, file);
+ if (expectedHash is not null && !string.Equals(upstreamHash, expectedHash, StringComparison.Ordinal))
+ {
+ throw new InvalidDataException("Downloaded checkpoint does not match its pinned upstream digest.");
+ }
+
+ if (file == "tokenizer/tokenizer_config.json")
+ {
+ await NormalizeTokenizerConfigAsync(temporary, cancellationToken);
+ }
+
+ File.Move(temporary, target, overwrite: true);
+ }
+ finally
+ {
+ if (File.Exists(temporary)) File.Delete(temporary);
+ }
+ }
+
+ private static async Task NormalizeTokenizerConfigAsync(string path, CancellationToken cancellationToken)
+ {
+ var text = await File.ReadAllTextAsync(path, cancellationToken);
+ var config = JsonNode.Parse(text) as JsonObject ?? throw new InvalidDataException("Tokenizer configuration is invalid.");
+ var tokenizerClass = config["tokenizer_class"]?.GetValue();
+ if (tokenizerClass is null or "TokenizersBackend")
+ {
+ config["tokenizer_class"] = "PreTrainedTokenizerFast";
+ config.Remove("backend");
+ config.Remove("is_local");
+ }
+
+ if (config["extra_special_tokens"] is JsonArray specialTokens)
+ {
+ var normalized = new JsonObject();
+ for (var index = 0; index < specialTokens.Count; index++)
+ {
+ normalized[$"extra_{index}"] = specialTokens[index]?.DeepClone();
+ }
+ config["extra_special_tokens"] = normalized;
+ }
+
+ var normalizedBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(config, JsonOptions) + "\n");
+ await File.WriteAllBytesAsync(path, normalizedBytes, cancellationToken);
+ }
+
+ private static async Task HashFileAsync(string path, CancellationToken cancellationToken)
+ {
+ await using var stream = File.OpenRead(path);
+ return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken));
+ }
+
+ private static void CopyProvenanceFile(string sourceRelativePath, string destination)
+ {
+ var source = Path.Combine(AppContext.BaseDirectory, sourceRelativePath.Replace('/', Path.DirectorySeparatorChar));
+ if (!File.Exists(source))
+ {
+ var repositoryRoot = FindRepositoryRoot();
+ source = Path.Combine(repositoryRoot, "tools", "laya_service", sourceRelativePath.Replace('/', Path.DirectorySeparatorChar));
+ }
+ File.Copy(source, destination, overwrite: true);
+ }
+
+ private static string FindRepositoryRoot()
+ {
+ var directory = new DirectoryInfo(AppContext.BaseDirectory);
+ while (directory is not null)
+ {
+ if (File.Exists(Path.Combine(directory.FullName, "OpenClaw.Net.slnx"))) return directory.FullName;
+ directory = directory.Parent;
+ }
+ throw new DirectoryNotFoundException("Laya attribution files are not available.");
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Models/ModelManifest.cs b/tools/laya_service/Models/ModelManifest.cs
new file mode 100644
index 00000000..8f2b67e5
--- /dev/null
+++ b/tools/laya_service/Models/ModelManifest.cs
@@ -0,0 +1,210 @@
+using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace OpenClaw.LayaService.Models;
+
+public sealed record VerifiedCheckpoint(
+ string Name,
+ string Revision,
+ string AbsolutePath,
+ IReadOnlyDictionary FileHashes);
+
+public sealed record VerifiedManifest(
+ string ManifestPath,
+ string Revision,
+ IReadOnlyDictionary Checkpoints);
+
+public sealed record ModelManifestDocument
+{
+ [JsonPropertyName("version")]
+ public required int Version { get; init; }
+
+ [JsonPropertyName("revision")]
+ public required string Revision { get; init; }
+
+ [JsonPropertyName("checkpoints")]
+ public required Dictionary Checkpoints { get; init; }
+}
+
+public sealed record CheckpointManifestEntry
+{
+ [JsonPropertyName("path")]
+ public required string Path { get; init; }
+
+ [JsonPropertyName("sha256")]
+ public required Dictionary Sha256 { get; init; }
+}
+
+public static class ModelManifest
+{
+ public const string DefaultRevision = "1c5edc17a7acd8701df6fc341c0d179f1c62c982";
+ public static IReadOnlyList ModelFiles { get; } = Array.AsReadOnly(new[]
+ {
+ "model.safetensors",
+ "rl_agent_config.json",
+ "encoder/config.json",
+ "tokenizer/tokenizer.json",
+ "tokenizer/tokenizer_config.json"
+ });
+ public static IReadOnlySet CheckpointNames { get; } = new HashSet(StringComparer.Ordinal)
+ {
+ "english", "multilingual", "typed-decisions"
+ };
+
+ private static readonly IReadOnlyDictionary> DefaultHashes =
+ new Dictionary>(StringComparer.Ordinal)
+ {
+ ["english"] = new Dictionary(StringComparer.Ordinal)
+ {
+ ["model.safetensors"] = "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
+ ["rl_agent_config.json"] = "ae287b56bbcf5f8c4f4541ae9dfd00c914c4c48b940b8398c3058af37ba92bbd",
+ ["encoder/config.json"] = "bf3ab80598fdccf414855a2ce80f22859e4492d06ca8a62ddd1cfb63972f8979",
+ ["tokenizer/tokenizer.json"] = "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
+ ["tokenizer/tokenizer_config.json"] = "50044de60daaa73df97d262e15a40d4faf0160e7d742df64b377877a1320dd12"
+ },
+ ["multilingual"] = new Dictionary(StringComparer.Ordinal)
+ {
+ ["model.safetensors"] = "9d628fd971b700382ac6f65920a86f149777b2e748e0c955fb3b19695aa8f204",
+ ["rl_agent_config.json"] = "25061739243b617ad88d1219ba6f8a9c86c5881ca28df024fa2d9b3b2fcc30c6",
+ ["encoder/config.json"] = "83f6916d13ef0f556ac461f28308dc2bffa7ebeadee8ec9e2db5812020ea5bb4",
+ ["tokenizer/tokenizer.json"] = "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f",
+ ["tokenizer/tokenizer_config.json"] = "6c6b2d8e3c84ce0e671c129cd6b374b235d6f9863042a5836358d00a89bbb5a1"
+ },
+ ["typed-decisions"] = new Dictionary(StringComparer.Ordinal)
+ {
+ ["model.safetensors"] = "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e",
+ ["rl_agent_config.json"] = "ebf0cd524d92342a6be5e48e9fca3d7c2babfb5a56ccd79d2171ef5d8c7f7be8",
+ ["encoder/config.json"] = "5268d24ad3b77c8151de5dcb0762ba4391619aad9ab0bda33e36fb083cfeae6d",
+ ["tokenizer/tokenizer.json"] = "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
+ ["tokenizer/tokenizer_config.json"] = "08d4cf3ac4dca381759441b85b91a6d40e688471dcd33d15d6649eb0a9a854d1"
+ }
+ };
+
+ public static VerifiedManifest LoadAndVerify(string manifestPath, string expectedRevision)
+ => LoadAndVerify(manifestPath, expectedRevision, requireExpectedRevision: true);
+
+ public static VerifiedManifest LoadAndVerify(string manifestPath)
+ => LoadAndVerify(manifestPath, expectedRevision: null, requireExpectedRevision: false);
+
+ private static VerifiedManifest LoadAndVerify(string manifestPath, string? expectedRevision, bool requireExpectedRevision)
+ {
+ try
+ {
+ if (requireExpectedRevision && !IsRevision(expectedRevision)) throw InvalidManifest();
+ var fullManifestPath = System.IO.Path.GetFullPath(manifestPath);
+ var root = System.IO.Path.GetDirectoryName(fullManifestPath) ?? throw InvalidManifest();
+ using var document = JsonDocument.Parse(File.ReadAllBytes(fullManifestPath), new JsonDocumentOptions { MaxDepth = 32 });
+ RejectDuplicateProperties(document.RootElement);
+ var manifest = document.RootElement.Deserialize() ?? throw InvalidManifest();
+ if (manifest.Version != 1 || !IsRevision(manifest.Revision) ||
+ (expectedRevision is not null && !string.Equals(manifest.Revision, expectedRevision, StringComparison.Ordinal)) ||
+ manifest.Checkpoints is null || manifest.Checkpoints.Count == 0)
+ {
+ throw InvalidManifest();
+ }
+
+ var verified = new Dictionary(StringComparer.Ordinal);
+ foreach (var (name, entry) in manifest.Checkpoints)
+ {
+ if (!CheckpointNames.Contains(name) || entry?.Path is null || entry.Sha256 is null ||
+ !HasExactFiles(entry.Sha256.Keys))
+ {
+ throw InvalidManifest();
+ }
+
+ var directory = ResolveContainedPath(root, entry.Path);
+ if (!Directory.Exists(directory)) throw InvalidManifest();
+ RejectReparsePoints(root, directory);
+ foreach (var file in ModelFiles)
+ {
+ var expectedHash = entry.Sha256[file];
+ if (!IsHash(expectedHash)) throw InvalidManifest();
+ var filePath = ResolveContainedPath(directory, file);
+ if (!File.Exists(filePath)) throw InvalidManifest();
+ RejectReparsePoints(root, filePath);
+ using var stream = File.OpenRead(filePath);
+ var actualHash = Convert.ToHexStringLower(SHA256.HashData(stream));
+ if (!CryptographicOperations.FixedTimeEquals(Convert.FromHexString(actualHash), Convert.FromHexString(expectedHash)))
+ {
+ throw InvalidManifest();
+ }
+ }
+
+ verified.Add(name, new VerifiedCheckpoint(name, manifest.Revision, directory,
+ new Dictionary(entry.Sha256, StringComparer.Ordinal)));
+ }
+
+ return new VerifiedManifest(fullManifestPath, manifest.Revision, verified);
+ }
+ catch (InvalidDataException)
+ {
+ throw;
+ }
+ catch (Exception exception) when (exception is IOException or UnauthorizedAccessException or JsonException or ArgumentException or NotSupportedException)
+ {
+ throw InvalidManifest();
+ }
+ }
+
+ internal static bool IsRevision(string? revision)
+ => revision is { Length: 40 } && revision.All(character => character is >= '0' and <= '9' or >= 'a' and <= 'f');
+
+ internal static string? GetPinnedHash(string revision, string checkpoint, string file)
+ => revision == DefaultRevision && DefaultHashes.TryGetValue(checkpoint, out var checkpointHashes) &&
+ checkpointHashes.TryGetValue(file, out var hash) ? hash : null;
+
+ internal static bool HasExactFiles(IEnumerable names)
+ {
+ var actual = names.ToHashSet(StringComparer.Ordinal);
+ return actual.Count == ModelFiles.Count && ModelFiles.All(actual.Contains);
+ }
+
+ internal static string ResolveContainedPath(string root, string relativePath)
+ {
+ if (string.IsNullOrWhiteSpace(relativePath) || System.IO.Path.IsPathRooted(relativePath)) throw InvalidManifest();
+ var fullRoot = System.IO.Path.GetFullPath(root);
+ var fullPath = System.IO.Path.GetFullPath(System.IO.Path.Combine(fullRoot, relativePath));
+ var relative = System.IO.Path.GetRelativePath(fullRoot, fullPath);
+ if (relative == "." || relative == ".." || relative.StartsWith(".." + System.IO.Path.DirectorySeparatorChar, StringComparison.Ordinal) ||
+ relative.StartsWith(".." + System.IO.Path.AltDirectorySeparatorChar, StringComparison.Ordinal) || System.IO.Path.IsPathRooted(relative))
+ {
+ throw InvalidManifest();
+ }
+ return fullPath;
+ }
+
+ internal static bool IsHash(string? hash)
+ => hash is { Length: 64 } && hash.All(character => character is >= '0' and <= '9' or >= 'a' and <= 'f');
+
+ internal static void RejectReparsePoints(string root, string target)
+ {
+ var relative = System.IO.Path.GetRelativePath(System.IO.Path.GetFullPath(root), System.IO.Path.GetFullPath(target));
+ var current = System.IO.Path.GetFullPath(root);
+ foreach (var segment in relative.Split([System.IO.Path.DirectorySeparatorChar, System.IO.Path.AltDirectorySeparatorChar], StringSplitOptions.RemoveEmptyEntries))
+ {
+ current = System.IO.Path.Combine(current, segment);
+ var info = Directory.Exists(current) ? (FileSystemInfo)new DirectoryInfo(current) : new FileInfo(current);
+ if ((info.Attributes & FileAttributes.ReparsePoint) != 0) throw InvalidManifest();
+ }
+ }
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name)) throw InvalidManifest();
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray()) RejectDuplicateProperties(item);
+ }
+ }
+
+ private static InvalidDataException InvalidManifest() => new("Invalid model manifest or checkpoint assets.");
+}
\ No newline at end of file
diff --git a/tools/laya_service/Program.cs b/tools/laya_service/Program.cs
new file mode 100644
index 00000000..c693020f
--- /dev/null
+++ b/tools/laya_service/Program.cs
@@ -0,0 +1,167 @@
+using System.Net;
+using System.Globalization;
+using System.Text.Json;
+using OpenClaw.LayaService.Evaluation;
+using OpenClaw.LayaService.Hosting;
+using OpenClaw.LayaService.Inference;
+using OpenClaw.LayaService.Models;
+using OpenClaw.LayaService.Reporting;
+
+namespace OpenClaw.LayaService;
+
+public static class Program
+{
+ public static async Task Main(string[] args)
+ {
+ CommandInvocation? invocation = null;
+ try
+ {
+ invocation = CommandLine.Parse(args);
+ if (invocation.Command == "download")
+ {
+ var destination = invocation.Options.Get("destination")
+ ?? throw new ArgumentException("A destination is required.");
+ var revision = invocation.Options.Get("revision") ?? ModelManifest.DefaultRevision;
+ var handler = new HttpClientHandler { AllowAutoRedirect = false, UseProxy = true };
+ using var http = new HttpClient(handler, disposeHandler: true);
+ var manifestPath = await HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(destination, revision, invocation.Options.GetMany("checkpoint")), http, CancellationToken.None);
+ Console.WriteLine(manifestPath);
+ return 0;
+ }
+
+ if (invocation.Command == "serve")
+ {
+ var manifestPath = invocation.Options.Get("manifest")
+ ?? throw new ArgumentException("A manifest is required.");
+ var manifest = ModelManifest.LoadAndVerify(manifestPath);
+ var options = new ServeOptions(
+ manifestPath,
+ invocation.Options.Get("calibration"),
+ ParseIntOption(invocation.Options.Get("port"), 8099, 1, 65535),
+ invocation.Options.Get("device") ?? "cpu",
+ invocation.Options.Get("checkpoint") ?? "auto",
+ ParseIntOption(invocation.Options.Get("threads"), 4, 1, 256));
+ await using var predictor = await NLayaDecisionPredictor.LoadAsync(manifest, options, CancellationToken.None);
+ await using var app = DecisionServer.Build(predictor,
+ new ServiceOptions(options.Port, 16, 65536, TimeSpan.FromSeconds(60)));
+ await app.RunAsync();
+ return 0;
+ }
+
+ if (invocation.Command == "evaluate")
+ {
+ if (invocation.Arguments.Count != 1) throw new ArgumentException("An evaluation dataset is required.");
+ var output = invocation.Options.Get("output") ?? throw new ArgumentException("An output path is required.");
+ var endpoint = new Uri(invocation.Options.Get("endpoint") ?? "http://127.0.0.1:8099/v1/decisions", UriKind.Absolute);
+ var handler = new HttpClientHandler { AllowAutoRedirect = false, UseProxy = false };
+ using var http = new HttpClient(handler, disposeHandler: true);
+ http.Timeout = TimeSpan.FromSeconds(30);
+ var count = await CaseEvaluator.EvaluateAsync(invocation.Arguments[0], endpoint, output, http, CancellationToken.None);
+ Console.WriteLine(count.ToString(CultureInfo.InvariantCulture));
+ return 0;
+ }
+
+ if (invocation.Command == "calibrate")
+ {
+ var fitPath = invocation.Options.Get("fit") ?? throw new ArgumentException("A training observations path is required.");
+ var validationPath = invocation.Options.Get("validate") ?? throw new ArgumentException("A validation observations path is required.");
+ var outputPath = invocation.Options.Get("output") ?? throw new ArgumentException("An output path is required.");
+ var training = await CalibrationFitter.ReadObservationsAsync(fitPath, CancellationToken.None);
+ var validation = await CalibrationFitter.ReadObservationsAsync(validationPath, CancellationToken.None);
+ var artifact = CalibrationFitter.Fit(training, validation);
+ Console.WriteLine(await CalibrationFitter.WriteAsync(artifact, outputPath, CancellationToken.None));
+ return 0;
+ }
+
+ if (invocation.Command == "report")
+ {
+ if (invocation.Arguments.Count != 1) throw new ArgumentException("A routing journal path is required.");
+ var rows = RoutingJournalReport.ReadJsonLines(invocation.Arguments[0]);
+ var labelPath = invocation.Options.Get("labels");
+ var labels = labelPath is null ? Array.Empty() : RoutingJournalReport.ReadJsonLines(labelPath);
+ var report = RoutingJournalReport.Summarize(rows, labels);
+ var plotPath = invocation.Options.Get("plot");
+ if (plotPath is not null) ReliabilityPlot.WritePng(report, plotPath);
+ var rendered = report.Document.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) + "\n";
+ var outputPath = invocation.Options.Get("output");
+ if (outputPath is null)
+ {
+ Console.Write(rendered);
+ }
+ else
+ {
+ await WriteAtomicTextAsync(outputPath, rendered, CancellationToken.None);
+ }
+ return 0;
+ }
+
+ Console.Error.WriteLine($"{invocation.Command}_not_implemented");
+ return 2;
+ }
+ catch (ArgumentException)
+ {
+ Console.Error.WriteLine("invalid_arguments");
+ return 2;
+ }
+ catch (InvalidDataException)
+ {
+ Console.Error.WriteLine(invocation?.Command switch
+ {
+ "evaluate" => "invalid_evaluation_data",
+ "calibrate" => "invalid_calibration_data",
+ "report" => "invalid_report_data",
+ _ => "invalid_model_assets"
+ });
+ return 2;
+ }
+ catch (HttpRequestException)
+ {
+ Console.Error.WriteLine(invocation?.Command == "evaluate" ? "evaluation_failed" : "download_failed");
+ return 1;
+ }
+ catch (OperationCanceledException)
+ {
+ Console.Error.WriteLine("operation_cancelled");
+ return 1;
+ }
+ catch (InvalidOperationException exception) when (exception.Message is
+ "checkpoint_not_installed" or "requested_device_unavailable" or "unknown_checkpoint")
+ {
+ Console.Error.WriteLine(exception.Message);
+ return 2;
+ }
+ catch
+ {
+ Console.Error.WriteLine("command_failed");
+ return 1;
+ }
+ }
+
+ private static int ParseIntOption(string? value, int defaultValue, int minimum, int maximum)
+ {
+ if (value is null) return defaultValue;
+ if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var result) || result < minimum || result > maximum)
+ {
+ throw new ArgumentException("Invalid numeric option.");
+ }
+ return result;
+ }
+
+ private static async Task WriteAtomicTextAsync(string outputPath, string content, CancellationToken cancellationToken)
+ {
+ var fullPath = Path.GetFullPath(outputPath);
+ var directory = Path.GetDirectoryName(fullPath) ?? throw new ArgumentException("Invalid output path.");
+ Directory.CreateDirectory(directory);
+ var temporary = Path.Combine(directory, "." + Path.GetFileName(fullPath) + "." + Guid.NewGuid().ToString("N") + ".tmp");
+ try
+ {
+ await File.WriteAllTextAsync(temporary, content, new System.Text.UTF8Encoding(false), cancellationToken);
+ File.Move(temporary, fullPath, overwrite: true);
+ }
+ finally
+ {
+ if (File.Exists(temporary)) File.Delete(temporary);
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Protocol/RequestValidator.cs b/tools/laya_service/Protocol/RequestValidator.cs
new file mode 100644
index 00000000..8edc7f06
--- /dev/null
+++ b/tools/laya_service/Protocol/RequestValidator.cs
@@ -0,0 +1,150 @@
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using System.Text.RegularExpressions;
+
+namespace OpenClaw.LayaService.Protocol;
+
+public static class RequestValidator
+{
+ private static readonly HashSet Checkpoints = ["english", "multilingual", "typed-decisions"];
+ private static readonly HashSet QuestionProperties = ["type", "instructions", "criteria"];
+
+ public static void Validate(DecisionWireRequest request, string configuredModel)
+ {
+ if (!string.Equals(request.Model, configuredModel, StringComparison.Ordinal))
+ {
+ Reject("model_version_mismatch");
+ }
+
+ if (request.State.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array) ||
+ StrictJson.Canonicalize(request.State).Length > 32000)
+ {
+ Reject("invalid_state");
+ }
+
+ if (request.RubricVersion is null || !Regex.IsMatch(request.RubricVersion, "^[a-zA-Z0-9_.-]{1,80}$", RegexOptions.CultureInvariant))
+ {
+ Reject("invalid_rubric");
+ }
+
+ if (request.Language is not null && !Regex.IsMatch(request.Language, "^[a-zA-Z0-9-]{1,35}$", RegexOptions.CultureInvariant))
+ {
+ Reject("invalid_language");
+ }
+
+ if (request.Questions.ValueKind != JsonValueKind.Object || request.Questions.GetPropertyCount() is < 1 or > 16)
+ {
+ Reject("invalid_questions");
+ }
+
+ foreach (var question in request.Questions.EnumerateObject())
+ {
+ ValidateQuestion(question.Name, question.Value);
+ }
+ }
+
+ private static void ValidateQuestion(string name, JsonElement question)
+ {
+ if (!Regex.IsMatch(name, "^[a-zA-Z0-9_.-]{1,80}$", RegexOptions.CultureInvariant) || question.ValueKind != JsonValueKind.Object)
+ {
+ Reject("invalid_question");
+ }
+
+ if (question.EnumerateObject().Any(property => !QuestionProperties.Contains(property.Name)))
+ {
+ Reject("invalid_question");
+ }
+
+ if (!question.TryGetProperty("instructions", out var instructions) || instructions.ValueKind != JsonValueKind.String ||
+ string.IsNullOrEmpty(instructions.GetString()) || instructions.GetString()!.Length > 2000)
+ {
+ Reject("invalid_instructions");
+ }
+
+ if (!question.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String)
+ {
+ Reject("invalid_question_type");
+ }
+
+ var criteria = question.TryGetProperty("criteria", out var criteriaValue) ? criteriaValue : default;
+
+ switch (type.GetString())
+ {
+ case "choice":
+ ValidateChoice(criteria);
+ break;
+ case "score":
+ ValidateScore(criteria);
+ break;
+ case "noul":
+ ValidateNoul(criteria);
+ break;
+ default:
+ Reject("invalid_question_type");
+ break;
+ }
+ }
+
+ private static void ValidateChoice(JsonElement criteria)
+ {
+ if (criteria.ValueKind != JsonValueKind.Object || criteria.GetPropertyCount() is < 2 or > 20)
+ {
+ Reject("invalid_choices");
+ }
+
+ foreach (var item in criteria.EnumerateObject())
+ {
+ if (item.Name.Length is < 1 or > 80 || !IsOptionalText(item.Value, 2000))
+ {
+ Reject("invalid_criteria");
+ }
+ }
+ }
+
+ private static void ValidateScore(JsonElement criteria)
+ {
+ if (criteria.ValueKind != JsonValueKind.Array || criteria.GetArrayLength() is < 2 or > 10)
+ {
+ Reject("invalid_score");
+ }
+
+ foreach (var item in criteria.EnumerateArray())
+ {
+ if (!IsOptionalText(item, 2000))
+ {
+ Reject("invalid_criteria");
+ }
+ }
+ }
+
+ private static void ValidateNoul(JsonElement criteria)
+ {
+ if (criteria.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined) return;
+ if (criteria.ValueKind != JsonValueKind.Object || criteria.GetPropertyCount() != 2)
+ {
+ Reject("invalid_noul");
+ }
+
+ if (!criteria.TryGetProperty("false", out var falseValue))
+ {
+ Reject("invalid_noul");
+ }
+
+ if (!criteria.TryGetProperty("true", out var trueValue))
+ {
+ Reject("invalid_noul");
+ }
+
+ if (!IsOptionalText(falseValue, 2000) || !IsOptionalText(trueValue, 2000))
+ {
+ Reject("invalid_criteria");
+ }
+ }
+
+ private static bool IsOptionalText(JsonElement value, int maximumLength)
+ => value.ValueKind == JsonValueKind.Null ||
+ (value.ValueKind == JsonValueKind.String && value.GetString()!.Length <= maximumLength);
+
+ [DoesNotReturn]
+ private static void Reject(string reasonCode) => throw new ProtocolRejectionException(reasonCode);
+}
\ No newline at end of file
diff --git a/tools/laya_service/Protocol/StrictJson.cs b/tools/laya_service/Protocol/StrictJson.cs
new file mode 100644
index 00000000..633d76ed
--- /dev/null
+++ b/tools/laya_service/Protocol/StrictJson.cs
@@ -0,0 +1,139 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using OpenClaw.LayaService.Protocol;
+
+namespace OpenClaw.LayaService.Protocol;
+
+public static class StrictJson
+{
+ private static readonly HashSet RequestProperties =
+ ["model", "state", "questions", "rubric_version", "language"];
+
+ public static DecisionWireRequest ParseRequest(ReadOnlySpan utf8Json)
+ {
+ try
+ {
+ using var document = JsonDocument.Parse(utf8Json.ToArray(), new JsonDocumentOptions { MaxDepth = 64 });
+ if (document.RootElement.ValueKind != JsonValueKind.Object)
+ {
+ throw new ProtocolRejectionException("invalid_request");
+ }
+
+ RejectDuplicateProperties(document.RootElement);
+ if (document.RootElement.EnumerateObject().Any(property => !RequestProperties.Contains(property.Name)))
+ {
+ throw new ProtocolRejectionException("invalid_request");
+ }
+
+ return document.RootElement.Deserialize()
+ ?? throw new ProtocolRejectionException("invalid_request");
+ }
+ catch (ProtocolRejectionException)
+ {
+ throw;
+ }
+ catch (JsonException)
+ {
+ throw new ProtocolRejectionException("invalid_json");
+ }
+ catch (DecoderFallbackException)
+ {
+ throw new ProtocolRejectionException("invalid_json");
+ }
+ }
+
+ public static string Canonicalize(JsonElement value)
+ {
+ var output = new StringBuilder();
+ AppendCanonical(value, output);
+ return output.ToString();
+ }
+
+ public static string SchemaHash(JsonElement questions)
+ => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(Canonicalize(questions))));
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name))
+ {
+ throw new ProtocolRejectionException("invalid_json");
+ }
+
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray())
+ {
+ RejectDuplicateProperties(item);
+ }
+ }
+ }
+
+ private static void AppendCanonical(JsonElement value, StringBuilder output)
+ {
+ switch (value.ValueKind)
+ {
+ case JsonValueKind.Object:
+ output.Append('{');
+ var firstProperty = true;
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!firstProperty) output.Append(',');
+ firstProperty = false;
+ AppendString(property.Name, output);
+ output.Append(':');
+ AppendCanonical(property.Value, output);
+ }
+ output.Append('}');
+ break;
+ case JsonValueKind.Array:
+ output.Append('[');
+ var firstItem = true;
+ foreach (var item in value.EnumerateArray())
+ {
+ if (!firstItem) output.Append(',');
+ firstItem = false;
+ AppendCanonical(item, output);
+ }
+ output.Append(']');
+ break;
+ case JsonValueKind.String:
+ AppendString(value.GetString()!, output);
+ break;
+ default:
+ output.Append(value.GetRawText());
+ break;
+ }
+ }
+
+ private static void AppendString(string value, StringBuilder output)
+ {
+ output.Append('"');
+ foreach (var character in value)
+ {
+ switch (character)
+ {
+ case '"': output.Append("\\\""); break;
+ case '\\': output.Append("\\\\"); break;
+ case '\b': output.Append("\\b"); break;
+ case '\f': output.Append("\\f"); break;
+ case '\n': output.Append("\\n"); break;
+ case '\r': output.Append("\\r"); break;
+ case '\t': output.Append("\\t"); break;
+ default:
+ if (character < 0x20) output.Append("\\u").Append(((int)character).ToString("x4"));
+ else output.Append(character);
+ break;
+ }
+ }
+ output.Append('"');
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/Protocol/WireModels.cs b/tools/laya_service/Protocol/WireModels.cs
new file mode 100644
index 00000000..469c766e
--- /dev/null
+++ b/tools/laya_service/Protocol/WireModels.cs
@@ -0,0 +1,27 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace OpenClaw.LayaService.Protocol;
+
+public sealed record DecisionWireRequest
+{
+ [JsonPropertyName("model")]
+ public required string Model { get; init; }
+
+ [JsonPropertyName("state")]
+ public required JsonElement State { get; init; }
+
+ [JsonPropertyName("questions")]
+ public required JsonElement Questions { get; init; }
+
+ [JsonPropertyName("rubric_version")]
+ public required string RubricVersion { get; init; }
+
+ [JsonPropertyName("language")]
+ public string? Language { get; init; }
+}
+
+public sealed class ProtocolRejectionException(string reasonCode) : Exception(reasonCode)
+{
+ public string ReasonCode { get; } = reasonCode;
+}
\ No newline at end of file
diff --git a/tools/laya_service/README.md b/tools/laya_service/README.md
index 2635a91c..3a6164ba 100644
--- a/tools/laya_service/README.md
+++ b/tools/laya_service/README.md
@@ -1,5 +1,7 @@
-# Local Laya service
+# Local Laya Service
-See [the operator guide](../../docs/laya-routing.md) for pinned model preparation, local serving, gateway configuration, evaluation, and rollback.
+This standalone .NET 10 CLI downloads pinned model assets, serves NLaya decisions over loopback HTTP, evaluates labeled cases, fits v2 calibration artifacts, and reports Jev/Laya routing journals. It uses `NLaya` and `NLaya.TorchSharp` 1.0.0 with `TorchSharp-cpu` 0.107.0. The Gateway remains a separate process and does not reference the inference packages.
-Laya is developed by Nandakishor Mukkunnoth, ConvAI Innovations, and upstream contributors. Read [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for author credit, research links, and the retained upstream license. This directory contains OpenClaw's adapter and compatibility improvements; the SDK and weights remain upstream artifacts.
+Start with the [operator guide](../../docs/laya-routing.md) for `download`, `serve`, `evaluate`, `calibrate`, and `report` commands, artifact handling, Gateway configuration, verification, and rollback. The service is currently documented for JIT deployment; NativeAOT support is not claimed. CPU is the verified backend with the checked-in package set.
+
+Laya is developed by Nandakishor Mukkunnoth, ConvAI Innovations, and upstream contributors. Read [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md) for author credit, research links, and the retained upstream license. The SDK and weights remain upstream artifacts.
diff --git a/tools/laya_service/Reporting/ReliabilityPlot.cs b/tools/laya_service/Reporting/ReliabilityPlot.cs
new file mode 100644
index 00000000..c0fbdec8
--- /dev/null
+++ b/tools/laya_service/Reporting/ReliabilityPlot.cs
@@ -0,0 +1,46 @@
+using ScottPlot;
+using ScottPlot.MultiplotLayouts;
+
+namespace OpenClaw.LayaService.Reporting;
+
+public static class ReliabilityPlot
+{
+ public static void WritePng(RoutingReport report, string outputPath)
+ {
+ ArgumentNullException.ThrowIfNull(report);
+ if (report.CalibrationCohorts.Count == 0) throw new InvalidDataException("Reliability plots require labeled probabilities.");
+ var multiplot = new Multiplot
+ {
+ Layout = new Grid(report.CalibrationCohorts.Count, 2)
+ };
+ for (var index = 0; index < report.CalibrationCohorts.Count; index++)
+ {
+ var cohort = report.CalibrationCohorts[index];
+ var reliability = index == 0 ? multiplot.Subplots.GetPlot(0) : multiplot.AddPlot();
+ reliability.Title(Title(cohort), 12);
+ reliability.Axes.Bottom.Label.Text = "Mean top probability";
+ reliability.Axes.Left.Label.Text = "Accuracy";
+ reliability.Axes.SetLimits(0, 1, 0, 1);
+ reliability.Add.Line(0, 0, 1, 1);
+ var bins = cohort.Metrics.ReliabilityBins;
+ reliability.Add.ScatterLine(bins.Select(bin => bin.MeanTopProbability).ToArray(), bins.Select(bin => bin.Accuracy).ToArray());
+
+ var risk = multiplot.AddPlot();
+ risk.Title(Title(cohort), 12);
+ risk.Axes.Bottom.Label.Text = "Coverage";
+ risk.Axes.Left.Label.Text = "Error rate";
+ risk.Axes.SetLimits(0, 1, 0, 1);
+ var points = cohort.Metrics.RiskCoverage.Where(point => point.ErrorRate.HasValue).ToArray();
+ risk.Add.ScatterLine(points.Select(point => point.Coverage).ToArray(), points.Select(point => point.ErrorRate!.Value).ToArray());
+ }
+
+ var fullPath = Path.GetFullPath(outputPath);
+ Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
+ multiplot.SavePng(fullPath, 1400, Math.Max(480, report.CalibrationCohorts.Count * 420));
+ }
+
+ private static string Title(CalibrationCohort cohort)
+ => $"{cohort.Provider} / {cohort.Checkpoint ?? cohort.Model ?? "unknown"}\n{cohort.RubricVersion ?? "unknown rubric"} | revision {Short(cohort.Revision)} | calibration {Short(cohort.CalibrationId)}";
+
+ private static string Short(string? value) => string.IsNullOrEmpty(value) ? "n/a" : value[..Math.Min(value.Length, 8)];
+}
\ No newline at end of file
diff --git a/tools/laya_service/Reporting/RoutingJournalReport.cs b/tools/laya_service/Reporting/RoutingJournalReport.cs
new file mode 100644
index 00000000..52902b8c
--- /dev/null
+++ b/tools/laya_service/Reporting/RoutingJournalReport.cs
@@ -0,0 +1,352 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using OpenClaw.LayaService.Evaluation;
+
+namespace OpenClaw.LayaService.Reporting;
+
+public sealed record CalibrationCohort(
+ string Provider,
+ string? Model,
+ string? RubricVersion,
+ string? Checkpoint,
+ string? Revision,
+ string? CalibrationId,
+ string? SchemaHash,
+ JsonArray Identity,
+ CalibrationMetricsResult Metrics);
+
+public sealed record RoutingReport(JsonObject Document, IReadOnlyList CalibrationCohorts);
+
+public static class RoutingJournalReport
+{
+ private static readonly string[] Tiers = ["T0", "T1", "T2", "T3"];
+
+ public static IReadOnlyList ReadJsonLines(string path)
+ {
+ var rows = new List();
+ using var reader = new StreamReader(path, System.Text.Encoding.UTF8, detectEncodingFromByteOrderMarks: false);
+ var lineNumber = 0;
+ while (reader.ReadLine() is { } line)
+ {
+ lineNumber++;
+ if (string.IsNullOrWhiteSpace(line)) continue;
+ try
+ {
+ using var document = JsonDocument.Parse(line, new JsonDocumentOptions { MaxDepth = 64 });
+ RejectDuplicateProperties(document.RootElement);
+ if (document.RootElement.ValueKind != JsonValueKind.Object)
+ throw new InvalidDataException($"{path}:{lineNumber}: expected an object");
+ rows.Add(document.RootElement.Clone());
+ }
+ catch (JsonException exception)
+ {
+ throw new InvalidDataException($"{path}:{lineNumber}: invalid JSON", exception);
+ }
+ }
+ return rows;
+ }
+
+ public static RoutingReport Summarize(IReadOnlyList rows, IReadOnlyList? labelRows = null)
+ {
+ ArgumentNullException.ThrowIfNull(rows);
+ if (rows.Count == 0) throw new InvalidDataException("The journal contains no decisions.");
+ var byId = new Dictionary(StringComparer.Ordinal);
+ var proposed = new List();
+ var completed = new List();
+ var latencies = new List(rows.Count);
+ var reportedTokens = 0.0;
+ var reportedCost = 0.0;
+ foreach (var row in rows)
+ {
+ if (row.ValueKind != JsonValueKind.Object) throw new InvalidDataException("Each decision must be an object.");
+ var id = RequiredString(row, "decision_id");
+ if (id.Length == 0 || !byId.TryAdd(id, row)) throw new InvalidDataException("Every decision must have a unique nonempty decision_id.");
+ if (!Tiers.Contains(StringValue(row, "baseline_tier"), StringComparer.Ordinal) ||
+ !Tiers.Contains(StringValue(row, "applied_tier"), StringComparer.Ordinal))
+ throw new InvalidDataException($"Invalid baseline/applied tier for {id}.");
+ var proposedTier = OptionalString(row, "proposed_tier");
+ if (proposedTier is not null && !Tiers.Contains(proposedTier, StringComparer.Ordinal))
+ throw new InvalidDataException($"Invalid proposed tier for {id}.");
+ if (proposedTier is not null) proposed.Add(row);
+
+ var latency = RequiredFiniteNumber(row, "latency_ms");
+ if (latency < 0) throw new InvalidDataException($"Invalid latency for {id}.");
+ latencies.Add(latency);
+ if (row.TryGetProperty("input_tokens", out var tokens) && tokens.ValueKind != JsonValueKind.Null)
+ {
+ var tokenCount = Number(tokens, $"Invalid input token count for {id}.");
+ if (tokenCount < 0) throw new InvalidDataException($"Invalid input token count for {id}.");
+ completed.Add(row);
+ reportedTokens += tokenCount;
+ }
+ if (row.TryGetProperty("estimated_cost_usd", out var cost) && cost.ValueKind != JsonValueKind.Null)
+ {
+ var amount = Number(cost, $"Invalid cost for {id}.");
+ if (amount < 0) throw new InvalidDataException($"Invalid cost for {id}.");
+ reportedCost += amount;
+ }
+ }
+
+ var labels = new Dictionary(StringComparer.Ordinal);
+ foreach (var label in labelRows ?? Array.Empty())
+ {
+ var id = RequiredString(label, "decision_id");
+ var expected = RequiredString(label, "expected_tier");
+ if (!byId.ContainsKey(id) || !Tiers.Contains(expected, StringComparer.Ordinal) || labels.ContainsKey(id))
+ throw new InvalidDataException("Labels must reference unique journal decision IDs and expected_tier T0 through T3.");
+ var highRisk = false;
+ if (label.TryGetProperty("high_risk", out var highRiskElement))
+ {
+ if (highRiskElement.ValueKind is not (JsonValueKind.True or JsonValueKind.False))
+ throw new InvalidDataException("high_risk must be a JSON boolean.");
+ highRisk = highRiskElement.GetBoolean();
+ }
+ labels.Add(id, (expected, highRisk));
+ }
+
+ var cohorts = BuildCalibrationCohorts(rows, labels);
+ var document = new JsonObject
+ {
+ ["decisions"] = rows.Count,
+ ["responses_with_usage"] = completed.Count,
+ ["eligible_proposals"] = proposed.Count,
+ ["proposal_coverage"] = proposed.Count / (double)rows.Count,
+ ["modes"] = CountBy(rows, "mode", "unknown"),
+ ["providers"] = CountBy(rows, "provider", "jev"),
+ ["models"] = CountBy(rows, "model", "unreported"),
+ ["rubric_versions"] = CountBy(rows, "rubric_version", "unknown"),
+ ["reasons"] = CountBy(rows, "reason", "unknown"),
+ ["proposed_tiers"] = CountValues(proposed.Select(row => RequiredString(row, "proposed_tier"))),
+ ["proposal_disagreement_with_baseline"] = proposed.Count == 0 ? null : proposed.Count(row => RequiredString(row, "proposed_tier") != RequiredString(row, "baseline_tier")) / (double)proposed.Count,
+ ["added_latency_ms"] = new JsonObject
+ {
+ ["p50"] = Percentile(latencies, 0.50),
+ ["p95"] = Percentile(latencies, 0.95),
+ ["max"] = latencies.Max()
+ },
+ ["reported_input_tokens"] = reportedTokens,
+ ["estimated_reported_decision_cost_usd"] = Math.Round(reportedCost, 8),
+ ["quality"] = labels.Count == 0 ? null : BuildQuality(
+ rows.Where(row => labels.ContainsKey(RequiredString(row, "decision_id"))).ToArray(), labels, rows.Count),
+ ["limitations"] = new JsonArray(
+ "Decision cost excludes failed calls without usage, downstream models, retries, and cache effects.",
+ "Tier labels do not measure task success or establish calibrated confidence.",
+ "With ONNX disabled, baseline tier T2 is a bookkeeping default; the actual configured model is unchanged.",
+ "The proposal includes confidence gates and safety floors; missing proposals fall back to the baseline.",
+ "Compare model/rubric cohorts separately before tuning thresholds."),
+ ["calibration_quality"] = BuildCalibrationJson(cohorts)
+ };
+
+ var quality = document["quality"] as JsonObject;
+ if (quality is not null && rows.Any(row => OptionalString(row, "provider") == "laya"))
+ {
+ quality["decision_with_fallback"] = quality["jev_with_fallback"]!.DeepClone();
+ quality.Remove("jev_with_fallback");
+ }
+ return new RoutingReport(document, cohorts);
+ }
+
+ private static JsonObject? BuildQuality(
+ IReadOnlyList rows,
+ IReadOnlyDictionary labels,
+ int totalDecisions)
+ {
+ var byPolicy = new JsonObject
+ {
+ ["baseline"] = MeasureQuality(rows, labels, row => RequiredString(row, "baseline_tier")),
+ ["always_t2"] = MeasureQuality(rows, labels, _ => "T2"),
+ ["jev_with_fallback"] = MeasureQuality(rows, labels, row => OptionalString(row, "proposed_tier") ?? RequiredString(row, "baseline_tier"))
+ };
+ byPolicy["labeled_decisions"] = rows.Count;
+ byPolicy["label_coverage"] = rows.Count / (double)totalDecisions;
+ return byPolicy;
+ }
+
+ private static JsonObject MeasureQuality(
+ IReadOnlyList rows,
+ IReadOnlyDictionary labels,
+ Func predict)
+ {
+ var confusion = Tiers.ToDictionary(tier => tier,
+ _ => Tiers.ToDictionary(tier => tier, _ => 0, StringComparer.Ordinal), StringComparer.Ordinal);
+ var correct = 0;
+ var under = 0;
+ var over = 0;
+ var highRiskCount = 0;
+ var highRiskRetained = 0;
+ foreach (var row in rows)
+ {
+ var label = labels[RequiredString(row, "decision_id")];
+ var predicted = predict(row);
+ confusion[label.ExpectedTier][predicted]++;
+ var expectedIndex = Array.IndexOf(Tiers, label.ExpectedTier);
+ var predictedIndex = Array.IndexOf(Tiers, predicted);
+ correct += predictedIndex == expectedIndex ? 1 : 0;
+ under += predictedIndex < expectedIndex ? 1 : 0;
+ over += predictedIndex > expectedIndex ? 1 : 0;
+ if (label.HighRisk)
+ {
+ highRiskCount++;
+ if (predictedIndex >= Math.Max(2, expectedIndex)) highRiskRetained++;
+ }
+ }
+
+ var perTier = new JsonObject();
+ var f1Total = 0.0;
+ foreach (var tier in Tiers)
+ {
+ var truePositive = confusion[tier][tier];
+ var falsePositive = Tiers.Where(other => other != tier).Sum(other => confusion[other][tier]);
+ var falseNegative = Tiers.Where(other => other != tier).Sum(other => confusion[tier][other]);
+ var support = confusion[tier].Values.Sum();
+ var denominator = 2.0 * truePositive + falsePositive + falseNegative;
+ var f1 = denominator == 0
+ ? 0
+ : 2.0 * truePositive / denominator;
+ f1Total += f1;
+ perTier[tier] = new JsonObject { ["support"] = support, ["f1"] = f1 };
+ }
+
+ var confusionJson = new JsonObject();
+ foreach (var tier in Tiers) confusionJson[tier] = CountValues(Tiers.Select(predicted => (predicted, confusion[tier][predicted])));
+ return new JsonObject
+ {
+ ["samples"] = rows.Count,
+ ["accuracy"] = correct / (double)rows.Count,
+ ["under_routing_rate"] = under / (double)rows.Count,
+ ["over_routing_rate"] = over / (double)rows.Count,
+ ["high_risk_samples"] = highRiskCount,
+ ["high_risk_capability_retention"] = highRiskCount == 0 ? null : highRiskRetained / (double)highRiskCount,
+ ["macro_f1"] = f1Total / Tiers.Length,
+ ["per_tier"] = perTier,
+ ["confusion"] = confusionJson
+ };
+ }
+
+ private static IReadOnlyList BuildCalibrationCohorts(
+ IReadOnlyList rows,
+ IReadOnlyDictionary labels)
+ {
+ var groups = new Dictionary Rows)>(StringComparer.Ordinal);
+ foreach (var row in rows)
+ {
+ var id = RequiredString(row, "decision_id");
+ if (!labels.TryGetValue(id, out var label) || !row.TryGetProperty("probabilities", out var probabilities) ||
+ probabilities.ValueKind != JsonValueKind.Object || probabilities.GetPropertyCount() == 0) continue;
+ var metadata = row.TryGetProperty("metadata", out var metadataElement) && metadataElement.ValueKind == JsonValueKind.Object
+ ? metadataElement
+ : default;
+ var provider = OptionalString(row, "provider") ?? "jev";
+ var model = OptionalString(row, "model");
+ var rubric = OptionalString(row, "rubric_version");
+ var checkpoint = OptionalString(metadata, "checkpoint");
+ var revision = OptionalString(metadata, "revision");
+ var calibrationId = OptionalString(metadata, "calibration_id");
+ var schemaHash = OptionalString(metadata, "schema_hash");
+ var identity = new JsonArray(provider, model, rubric, checkpoint, revision, calibrationId, schemaHash);
+ var identityKey = identity.ToJsonString();
+ if (!groups.TryGetValue(identityKey, out var group))
+ {
+ group = (identity, []);
+ groups.Add(identityKey, group);
+ }
+ var keys = probabilities.EnumerateObject().Select(property => property.Name).ToArray();
+ var values = probabilities.EnumerateObject().Select(property => Number(property.Value, "Invalid probability distribution.")).ToArray();
+ group.Rows.Add(new LabeledPrediction("choice", keys, values, label.ExpectedTier));
+ }
+
+ return groups.Select(pair =>
+ {
+ var identity = pair.Value.Identity;
+ var values = identity.Select(node => node?.GetValue()).ToArray();
+ return new CalibrationCohort(values[0]!, values[1], values[2], values[3], values[4], values[5], values[6],
+ identity, CalibrationMetrics.Measure(pair.Value.Rows));
+ }).ToArray();
+ }
+
+ private static JsonArray BuildCalibrationJson(IReadOnlyList cohorts)
+ {
+ var output = new JsonArray();
+ foreach (var cohort in cohorts)
+ {
+ var metrics = JsonSerializerNode(cohort.Metrics);
+ metrics["cohort"] = cohort.Identity.DeepClone();
+ output.Add(metrics);
+ }
+ return output;
+ }
+
+ private static JsonObject JsonSerializerNode(T value)
+ => JsonSerializer.SerializeToNode(value)!.AsObject();
+
+ private static JsonObject CountBy(IEnumerable rows, string property, string fallback)
+ => CountValues(rows.Select(row => OptionalString(row, property) ?? fallback));
+
+ private static JsonObject CountValues(IEnumerable values)
+ {
+ var counts = new Dictionary(StringComparer.Ordinal);
+ foreach (var value in values) counts[value] = counts.GetValueOrDefault(value) + 1;
+ var output = new JsonObject();
+ foreach (var pair in counts) output[pair.Key] = pair.Value;
+ return output;
+ }
+
+ private static JsonObject CountValues(IEnumerable<(string Key, int Count)> values)
+ {
+ var output = new JsonObject();
+ foreach (var (key, count) in values) output[key] = count;
+ return output;
+ }
+
+ private static double Percentile(IReadOnlyList values, double fraction)
+ {
+ var sorted = values.Order().ToArray();
+ return sorted[Math.Max(0, (int)Math.Ceiling(sorted.Length * fraction) - 1)];
+ }
+
+ private static string RequiredString(JsonElement value, string name)
+ {
+ if (value.ValueKind != JsonValueKind.Object || !value.TryGetProperty(name, out var property) || property.ValueKind != JsonValueKind.String)
+ throw new InvalidDataException($"Missing or invalid {name}.");
+ return property.GetString()!;
+ }
+
+ private static string StringValue(JsonElement value, string name)
+ => value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String ? property.GetString()! : string.Empty;
+
+ private static string? OptionalString(JsonElement value, string name)
+ => value.ValueKind == JsonValueKind.Object && value.TryGetProperty(name, out var property) && property.ValueKind == JsonValueKind.String
+ ? property.GetString()
+ : null;
+
+ private static double RequiredFiniteNumber(JsonElement value, string name)
+ {
+ if (!value.TryGetProperty(name, out var property)) throw new InvalidDataException($"Missing or invalid {name}.");
+ return Number(property, $"Invalid {name}.");
+ }
+
+ private static double Number(JsonElement value, string error)
+ {
+ if (value.ValueKind != JsonValueKind.Number || !value.TryGetDouble(out var number) || !double.IsFinite(number))
+ throw new InvalidDataException(error);
+ return number;
+ }
+
+ private static void RejectDuplicateProperties(JsonElement value)
+ {
+ if (value.ValueKind == JsonValueKind.Object)
+ {
+ var names = new HashSet(StringComparer.Ordinal);
+ foreach (var property in value.EnumerateObject())
+ {
+ if (!names.Add(property.Name)) throw new InvalidDataException("Duplicate JSON property.");
+ RejectDuplicateProperties(property.Value);
+ }
+ }
+ else if (value.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var item in value.EnumerateArray()) RejectDuplicateProperties(item);
+ }
+ }
+}
diff --git a/tools/laya_service/__init__.py b/tools/laya_service/__init__.py
deleted file mode 100644
index 8a53dd90..00000000
--- a/tools/laya_service/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-"""OpenClaw's optional local adapter for Laya by Nandakishor Mukkunnoth / ConvAI Innovations.
-
-Laya is an Apache-2.0 dependency. See THIRD_PARTY_NOTICES.md for attribution.
-"""
diff --git a/tools/laya_service/__main__.py b/tools/laya_service/__main__.py
deleted file mode 100644
index 819c630a..00000000
--- a/tools/laya_service/__main__.py
+++ /dev/null
@@ -1,140 +0,0 @@
-"""Serve preloaded Laya checkpoints on loopback. No model downloads at runtime."""
-import argparse
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-import json
-import socket
-import threading
-from .protocol import Rejected, canonical, read_json
-
-
-class DecisionServer(ThreadingHTTPServer):
- daemon_threads = True
- block_on_close = False
-
- def __init__(self, port, runtime):
- self.runtime = runtime
- self.inference = threading.Lock()
- self.connections = threading.BoundedSemaphore(8)
- super().__init__(("127.0.0.1", port), Handler)
-
- def process_request(self, request, client_address):
- if not self.connections.acquire(blocking=False):
- request.settimeout(1)
- try:
- request.sendall(b"HTTP/1.0 503 Service Unavailable\r\nContent-Length: 0\r\n\r\n")
- except OSError:
- # The overloaded peer may disconnect before receiving the response.
- pass
- self.shutdown_request(request)
- return
- try:
- super().process_request(request, client_address)
- except Exception:
- self.connections.release()
- raise
-
- def process_request_thread(self, request, client_address):
- try:
- super().process_request_thread(request, client_address)
- finally:
- self.connections.release()
-
- def handle_error(self, request, client_address):
- # No request bodies, headers, or exceptions in logs.
- pass
-
-
-class Handler(BaseHTTPRequestHandler):
- def setup(self):
- self.request.settimeout(10)
- super().setup()
-
- def log_message(self, *args):
- # Request metadata can contain sensitive local prompt information.
- pass
-
- def send_json(self, status, value):
- body = canonical(value).encode()
- self.send_response(status)
- self.send_header("Content-Type", "application/json")
- self.send_header("Content-Length", str(len(body)))
- self.send_header("Cache-Control", "no-store")
- self.end_headers()
- self.wfile.write(body)
-
- def local_request(self):
- if self.headers.get("Origin") is not None or self.headers.get("Host") != f"127.0.0.1:{self.server.server_port}":
- self.send_json(403, {"error": "local_clients_only"})
- return False
- return True
-
- def do_GET(self):
- if self.local_request():
- self.send_json(200, self.server.runtime.health()) if self.path == "/health" else self.send_json(404, {"error": "not_found"})
-
- def do_POST(self):
- if not self.local_request():
- return
- if self.path != "/v1/decisions":
- self.send_json(404, {"error": "not_found"})
- return
- if self.headers.get("Transfer-Encoding") or self.headers.get_content_type() != "application/json":
- self.send_json(415, {"error": "json_content_length_required"})
- return
- try:
- length = int(self.headers.get("Content-Length", "0"))
- except ValueError:
- length = 0
- if not 0 < length <= 65536:
- self.send_json(413, {"error": "request_size"})
- return
- if not self.server.inference.acquire(blocking=False):
- self.send_json(503, {"error": "busy"})
- return
- try:
- try:
- raw = self.rfile.read(length)
- if len(raw) != length:
- raise Rejected("incomplete_body")
- request = read_json(raw.decode("utf-8"))
- except (Rejected, UnicodeError, ValueError, TypeError, RecursionError) as exc:
- self.send_json(422, {"error": str(exc) if isinstance(exc, Rejected) else "invalid_request"})
- return
- try:
- result = self.server.runtime.predict(request)
- self.send_json(200, result)
- except Rejected as exc:
- self.send_json(422, {"error": str(exc)})
- except (socket.timeout, ConnectionError, BrokenPipeError):
- # The local client controls its own deadline and may close first.
- pass
- except Exception:
- self.send_json(503, {"error": "inference_failed"})
- finally:
- self.server.inference.release()
-
-
-def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--manifest", required=True)
- parser.add_argument("--calibration")
- parser.add_argument("--port", type=int, default=8099)
- parser.add_argument("--device", choices=("cpu", "mps", "cuda"), default="cpu")
- parser.add_argument("--checkpoint", choices=("auto", "english", "multilingual", "typed-decisions"), default="auto")
- parser.add_argument("--threads", type=int, default=4)
- args = parser.parse_args()
- if not 1 <= args.port <= 65535 or not 1 <= args.threads <= 32:
- parser.error("Invalid port or thread count.")
- from .runtime import Runtime
- runtime = Runtime(args.manifest, args.calibration, args.device, args.checkpoint, args.threads)
- with DecisionServer(args.port, runtime) as server:
- print(json.dumps({"listening": f"http://127.0.0.1:{args.port}", **runtime.health()}), flush=True)
- try:
- server.serve_forever()
- except KeyboardInterrupt:
- # Normal interactive shutdown.
- pass
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/laya_service/calibration.py b/tools/laya_service/calibration.py
deleted file mode 100644
index f2543ee1..00000000
--- a/tools/laya_service/calibration.py
+++ /dev/null
@@ -1,203 +0,0 @@
-"""Post-hoc temperature fitting and held-out evaluation; never modifies Laya weights.
-
-Inputs are local observations from evaluate.py. Fit and validation cases must be disjoint.
-"""
-import argparse
-import copy
-import hashlib
-import json
-import math
-from pathlib import Path
-from .protocol import CHECKPOINTS, SDK_VERSION, Rejected, distribution, file_hash, is_hex
-
-
-def temperature_scale(probabilities, temperature):
- logits = [math.log(max(p, 1e-9)) / temperature for p in probabilities]
- peak = max(logits)
- values = [math.exp(value - peak) for value in logits]
- total = sum(values)
- return [value / total for value in values]
-
-
-def confidence(probabilities, kind):
- if kind == "noul":
- return max(probabilities)
- return max(0.0, 1 + sum(p * math.log(max(p, 1e-12)) for p in probabilities) / math.log(len(probabilities)))
-
-
-def bucket(kind, count):
- return f"{kind}:{count}"
-
-
-def metrics(rows):
- count = len(rows)
- if not count:
- raise ValueError("No labeled predictions.")
- bins = [[] for _ in range(10)]
- nll = brier = correct_count = 0
- ranked = []
- for row in rows:
- keys, probabilities = distribution(row["answer"])
- expected = keys.index(row["label"])
- predicted = max(range(len(probabilities)), key=probabilities.__getitem__)
- correct = int(predicted == expected)
- top = probabilities[predicted]
- bins[min(9, int(top * 10))].append((top, correct))
- correct_count += correct
- nll -= math.log(max(probabilities[expected], 1e-9))
- brier += sum((p - int(i == expected)) ** 2 for i, p in enumerate(probabilities))
- ranked.append((confidence(probabilities, row["answer"]["type"]), correct))
- reliability = []
- ece = 0
- for i, values in enumerate(bins):
- if values:
- mean = sum(v[0] for v in values) / len(values)
- accuracy = sum(v[1] for v in values) / len(values)
- ece += len(values) / count * abs(mean - accuracy)
- reliability.append({"lower": i / 10, "upper": (i + 1) / 10,
- "samples": len(values), "mean_top_probability": mean, "accuracy": accuracy})
- # Include all ties; threshold curves must not pretend a tie can be selectively accepted.
- coverage = []
- for threshold in (0.0, 0.2, 0.4, 0.6, 0.8, 0.9, 0.95, 0.99):
- selected = [correct for conf, correct in ranked if conf >= threshold]
- coverage.append({"confidence_threshold": threshold, "coverage": len(selected) / count,
- "error_rate": 1 - sum(selected) / len(selected) if selected else None})
- return {"samples": count, "accuracy": correct_count / count, "nll": nll / count,
- "brier": brier / count, "ece": ece, "reliability_bins": reliability, "risk_coverage": coverage}
-
-
-def transform(answer, temperature):
- result = copy.deepcopy(answer)
- keys, values = distribution(answer)
- values = temperature_scale(values, temperature)
- result["confidence"] = confidence(values, answer["type"])
- if answer["type"] == "noul":
- result["noul"] = values[1]
- else:
- result["probabilities"] = dict(zip(keys, values))
- if answer["type"] == "choice":
- result["choice"] = keys[max(range(len(values)), key=values.__getitem__)]
- else:
- result["score"] = sum(int(key) * probability for key, probability in zip(keys, values))
- return result
-
-
-class Calibration:
- def __init__(self, path=None, model=None):
- self.data = None
- self.identifier = "uncalibrated"
- if path:
- payload = Path(path).read_bytes()
- data = json.loads(payload.decode("utf-8"))
- if (data.get("version") != 1 or data.get("sdk_version") != SDK_VERSION or
- data.get("model") != model or not is_hex(data.get("schema_hash"), 64) or
- not isinstance(data.get("temperatures"), dict) or not data["temperatures"] or
- not isinstance(data.get("validation"), dict)):
- raise ValueError("Invalid calibration artifact or model identity.")
- for checkpoint, values in data["temperatures"].items():
- if checkpoint not in CHECKPOINTS or not isinstance(values, dict) or not values:
- raise ValueError("Invalid calibration checkpoint.")
- for key, temperature in values.items():
- if not isinstance(key, str) or not isinstance(temperature, (int, float)) or not math.isfinite(temperature) or not 0.1 <= temperature <= 10:
- raise ValueError("Invalid calibration temperature.")
- self.data = data
- self.identifier = hashlib.sha256(payload).hexdigest()
-
- def check(self, schema, checkpoint, questions):
- if not self.data:
- return
- if self.data["schema_hash"] != schema:
- raise Rejected("calibration_schema_mismatch")
- values = self.data["temperatures"].get(checkpoint, {})
- for question in questions.values():
- count = 2 if question["type"] == "noul" else len(question["criteria"])
- if bucket(question["type"], count) not in values:
- raise Rejected("calibration_bucket_missing")
-
- def apply(self, checkpoint, answers):
- if not self.data:
- return answers
- result = {}
- for name, answer in answers.items():
- keys, _ = distribution(answer)
- result[name] = transform(answer, self.data["temperatures"][checkpoint][bucket(answer["type"], len(keys))])
- return result
-
-
-def load_observations(path):
- rows, seen = [], set()
- for line in Path(path).read_text().splitlines():
- if not line.strip():
- continue
- row = json.loads(line)
- identity = (row["case_id"], row["question_id"])
- if identity in seen or not all(isinstance(v, str) and v for v in identity):
- raise ValueError("Duplicate or empty observation identity.")
- seen.add(identity)
- if not is_hex(row.get("case_fingerprint"), 64) or row.get("checkpoint") not in CHECKPOINTS or not is_hex(row.get("schema_hash"), 64) or row.get("sdk_version") != SDK_VERSION or row.get("source_calibration") != "raw":
- raise ValueError("Observations must contain raw outputs and complete provenance.")
- if row["answer"].get("type") not in ("choice", "score", "noul"):
- raise ValueError("Unsupported answer type.")
- keys, _ = distribution(row["answer"])
- if row["label"] not in keys:
- raise ValueError("Label is outside the question's options.")
- rows.append(row)
- if not rows:
- raise ValueError("Empty observations.")
- return rows
-
-
-def fit(training, validation, minimum=20):
- if {r["case_id"] for r in training} & {r["case_id"] for r in validation}:
- raise ValueError("Calibration and validation case IDs overlap.")
- if {r["case_fingerprint"] for r in training} & {r["case_fingerprint"] for r in validation}:
- raise ValueError("Calibration and validation contain identical requests under different IDs.")
- identities = {(r["model"], r["schema_hash"], r["sdk_version"]) for r in training + validation}
- if len(identities) != 1:
- raise ValueError("Use one model revision, SDK version, and question schema per artifact.")
- model, schema, sdk = identities.pop()
- if not model.startswith("laya@") or not is_hex(model[5:], 40):
- raise ValueError("Unpinned model.")
- def group(row):
- keys, _ = distribution(row["answer"])
- return row["checkpoint"], bucket(row["answer"]["type"], len(keys))
- groups = {group(r) for r in training}
- if groups != {group(r) for r in validation}:
- raise ValueError("Training and validation must cover the same checkpoint/question buckets.")
- result = {"version": 1, "model": model, "schema_hash": schema, "sdk_version": sdk,
- "temperatures": {}, "validation": {}}
- for checkpoint, key in sorted(groups):
- train = [r for r in training if group(r) == (checkpoint, key)]
- test = [r for r in validation if group(r) == (checkpoint, key)]
- if min(len(train), len(test)) < minimum or len({r["label"] for r in train}) < 2:
- raise ValueError("Each bucket needs at least 20 calibration and 20 held-out observations, and multiple calibration labels.")
- def loss(temperature):
- total = 0
- for row in train:
- keys, values = distribution(row["answer"])
- total -= math.log(max(temperature_scale(values, temperature)[keys.index(row["label"])], 1e-9))
- return total
- temperature = min((10 ** (-1 + i / 80) for i in range(161)), key=loss)
- result["temperatures"].setdefault(checkpoint, {})[key] = temperature
- changed = [{**r, "answer": transform(r["answer"], temperature)} for r in test]
- result["validation"].setdefault(checkpoint, {})[key] = {
- "calibration_samples": len(train), "raw": metrics(test), "calibrated": metrics(changed)}
- return result
-
-
-def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--fit", required=True, help="Raw calibration observations JSONL")
- parser.add_argument("--validate", required=True, help="Disjoint held-out observations JSONL")
- parser.add_argument("--output", required=True)
- args = parser.parse_args()
- try:
- result = fit(load_observations(args.fit), load_observations(args.validate))
- Path(args.output).write_text(json.dumps(result, indent=2, allow_nan=False) + "\n")
- print(json.dumps({"calibration_id": file_hash(args.output), "validation": result["validation"]}))
- except (OSError, ValueError, KeyError, TypeError) as exc:
- parser.exit(2, str(exc) + "\n")
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/laya_service/compat.py b/tools/laya_service/compat.py
deleted file mode 100644
index b6882825..00000000
--- a/tools/laya_service/compat.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""Adapter improvements for Laya 0.3.4; no upstream files are patched at runtime."""
-import unicodedata
-from .protocol import Rejected
-
-
-def select_checkpoint(state, language=None):
- from laya.common import serialize_state
- from laya.lang import analyse
- text = serialize_state(state)
- # Laya 0.3.4 omits Armenian from its script ranges. Inspect every letter,
- # including minority scripts, so mixed text cannot silently use English.
- non_latin = any(c.isalpha() and "LATIN" not in unicodedata.name(c, "") for c in text)
- if non_latin:
- if language and language.lower().split("-")[0] in ("en", "eng", "english"):
- raise Rejected("language_script_conflict")
- return "multilingual"
- if language:
- return "english" if language.lower().split("-")[0] in ("en", "eng", "english") else "multilingual"
- detection = analyse(state)
- return "english" if detection["is_english"] else "multilingual"
-
-
-def ensure_complete(agent, state, questions):
- """Check the SDK's exact token construction before any silent truncation.
-
- Candidate descriptions have a separate per-option cap, and the question
- head has its own cap; checking only the overall state length is insufficient.
- """
- from laya.common import render_options, serialize_state
- tok = agent.tok
- mask = tok.mask_token
- def encode(text):
- if mask in text:
- # The SDK would replace marker strings; preserve exact input or reject.
- raise Rejected("reserved_token_in_input")
- return tok(text, add_special_tokens=False)["input_ids"]
- state_count = len(encode(serialize_state(state)))
- for q in questions.values():
- internal = agent._to_internal(q)
- options = render_options(internal)
- lengths = [1 + len(encode(" " + option)) for option in options]
- if any(length > 49 for length in lengths):
- raise Rejected("option_too_long")
- head_count = len(encode("%s question: %s" % (internal["t"], internal["ins"])))
- available = agent.cfg.get("head_max_len", 192) - sum(lengths)
- if available < 16 or head_count > max(8, available):
- raise Rejected("question_too_long")
- if 4 + head_count + sum(lengths) + state_count > agent.cfg.get("max_len", 512):
- raise Rejected("state_too_long")
diff --git a/tools/laya_service/download.py b/tools/laya_service/download.py
deleted file mode 100644
index 7127fa98..00000000
--- a/tools/laya_service/download.py
+++ /dev/null
@@ -1,93 +0,0 @@
-"""Download only selected, pinned Laya checkpoints and create an integrity manifest."""
-import argparse
-import json
-from pathlib import Path
-from .protocol import CHECKPOINTS, DEFAULT_REVISION, MODEL_FILES, file_hash, is_hex
-
-
-DEFAULT_FILE_HASHES = {
- "english": {
- "model.safetensors": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
- "rl_agent_config.json": "ae287b56bbcf5f8c4f4541ae9dfd00c914c4c48b940b8398c3058af37ba92bbd",
- "encoder/config.json": "bf3ab80598fdccf414855a2ce80f22859e4492d06ca8a62ddd1cfb63972f8979",
- "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
- "tokenizer/tokenizer_config.json": "50044de60daaa73df97d262e15a40d4faf0160e7d742df64b377877a1320dd12",
- },
- "multilingual": {
- "model.safetensors": "9d628fd971b700382ac6f65920a86f149777b2e748e0c955fb3b19695aa8f204",
- "rl_agent_config.json": "25061739243b617ad88d1219ba6f8a9c86c5881ca28df024fa2d9b3b2fcc30c6",
- "encoder/config.json": "83f6916d13ef0f556ac461f28308dc2bffa7ebeadee8ec9e2db5812020ea5bb4",
- "tokenizer/tokenizer.json": "609d8f4c067cd3950f88594c5a802616cea245823836ef5848ee4fc40aab5b6f",
- "tokenizer/tokenizer_config.json": "6c6b2d8e3c84ce0e671c129cd6b374b235d6f9863042a5836358d00a89bbb5a1",
- },
- "typed-decisions": {
- "model.safetensors": "4fa56de72383a9d3efa9cfa78955733c81b9fc8067a587ca4beb82c78107a24e",
- "rl_agent_config.json": "ebf0cd524d92342a6be5e48e9fca3d7c2babfb5a56ccd79d2171ef5d8c7f7be8",
- "encoder/config.json": "5268d24ad3b77c8151de5dcb0762ba4391619aad9ab0bda33e36fb083cfeae6d",
- "tokenizer/tokenizer.json": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
- "tokenizer/tokenizer_config.json": "08d4cf3ac4dca381759441b85b91a6d40e688471dcd33d15d6649eb0a9a854d1",
- },
-}
-
-
-def prepare(destination, revision, checkpoints):
- from huggingface_hub import snapshot_download
- if not is_hex(revision, 40):
- raise ValueError("Use an immutable 40-character Hugging Face commit revision.")
- destination = Path(destination).resolve()
- manifest_path = destination / "manifest.json"
- manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {
- "version": 1, "revision": revision, "checkpoints": {}}
- if manifest["revision"] != revision:
- raise ValueError("Use a separate destination for a different model revision.")
- destination.mkdir(parents=True, exist_ok=True)
- for checkpoint in checkpoints:
- if checkpoint not in CHECKPOINTS:
- raise ValueError("Unknown checkpoint.")
- prefix = "" if checkpoint == "english" else checkpoint + "/"
- snapshot_download("convaiinnovations/laya", revision=revision,
- allow_patterns=[prefix + name for name in MODEL_FILES],
- local_dir=destination / "hub")
- directory = destination / "hub" / prefix
- if revision == DEFAULT_REVISION:
- for name, expected in DEFAULT_FILE_HASHES[checkpoint].items():
- if file_hash(directory / name) != expected:
- raise ValueError("Downloaded checkpoint does not match the pinned upstream digest.")
- # Perform the SDK's tokenizer compatibility normalization during setup,
- # before hashing. The SDK must not rewrite an integrity-checked file at startup.
- tokenizer = directory / "tokenizer/tokenizer_config.json"
- config = json.loads(tokenizer.read_text())
- if config.get("tokenizer_class") in (None, "TokenizersBackend"):
- config["tokenizer_class"] = "PreTrainedTokenizerFast"
- config.pop("backend", None)
- config.pop("is_local", None)
- if isinstance(config.get("extra_special_tokens"), list):
- config["extra_special_tokens"] = {f"extra_{i}": value for i, value in enumerate(config["extra_special_tokens"])}
- tokenizer.write_text(json.dumps(config, indent=2) + "\n")
- manifest["checkpoints"][checkpoint] = {
- "path": str(directory.relative_to(destination)),
- "sha256": {name: file_hash(directory / name) for name in MODEL_FILES}}
- temporary = manifest_path.with_suffix(".tmp")
- temporary.write_text(json.dumps(manifest, indent=2) + "\n")
- temporary.replace(manifest_path)
- # Keep the model's provenance and upstream license beside downloaded assets.
- source = Path(__file__).parent
- (destination / "LAYA-NOTICE.md").write_text((source / "THIRD_PARTY_NOTICES.md").read_text())
- (destination / "LAYA-LICENSE.txt").write_text((source / "licenses/laya-APACHE-2.0.txt").read_text())
- return manifest_path
-
-
-def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--destination", required=True)
- parser.add_argument("--revision", default=DEFAULT_REVISION)
- parser.add_argument("--checkpoint", choices=CHECKPOINTS, action="append", required=True)
- args = parser.parse_args()
- try:
- print(prepare(args.destination, args.revision, args.checkpoint))
- except (ValueError, OSError) as exc:
- parser.exit(2, str(exc) + "\n")
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/laya_service/evaluate.py b/tools/laya_service/evaluate.py
deleted file mode 100644
index 801f475e..00000000
--- a/tools/laya_service/evaluate.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""Evaluate labeled local JSONL cases without recording state in observations.
-
-Each case: {case_id, model, state, questions, rubric_version, labels, language?}.
-Labels: choice key, score index, or a JSON boolean for noul. Separate datasets
-must be used for calibration fitting and validation.
-"""
-import argparse
-import hashlib
-import json
-from pathlib import Path
-from urllib.parse import urlparse
-from urllib.request import Request, build_opener, ProxyHandler, HTTPRedirectHandler
-from .protocol import DEFAULT_REVISION, canonical, distribution, read_json, validate_request
-
-
-class NoRedirect(HTTPRedirectHandler):
- def redirect_request(self, *args, **kwargs):
- return None
-
-
-def evaluate(dataset, endpoint, output):
- url = urlparse(endpoint)
- if url.scheme != "http" or url.hostname != "127.0.0.1" or url.username or url.password or url.query or url.fragment or url.path != "/v1/decisions":
- raise ValueError("Use the local http://127.0.0.1:PORT/v1/decisions endpoint.")
- opener = build_opener(ProxyHandler({}), NoRedirect())
- rows, seen = [], set()
- for line in Path(dataset).read_text().splitlines():
- if not line.strip():
- continue
- case = read_json(line)
- identifier = case.pop("case_id")
- labels = case.pop("labels")
- if not isinstance(identifier, str) or not identifier or identifier in seen:
- raise ValueError("Case IDs must be nonempty and unique.")
- seen.add(identifier)
- if "questions" not in case:
- rubric = read_json((Path(__file__).parent / "rubrics/openclaw-laya-tiers-v1.json").read_text())
- case.update(rubric)
- case.setdefault("model", "laya@" + DEFAULT_REVISION)
- questions = validate_request(case, case["model"])
- if set(labels) != set(questions):
- raise ValueError("Provide one label for every question.")
- request = Request(endpoint, data=canonical(case).encode(), headers={"Content-Type": "application/json"})
- with opener.open(request, timeout=30) as response:
- result = read_json(response.read(262145).decode())
- metadata = result["metadata"]
- if result["model"] != case["model"] or metadata["truncated"]:
- raise ValueError("Model identity mismatch or truncated prediction.")
- for name, answer in result["raw_answers"].items():
- label = labels[name]
- if answer["type"] == "noul":
- if not isinstance(label, bool):
- raise ValueError("Noul labels must be JSON booleans.")
- label = str(label).lower()
- else:
- label = str(label)
- keys, _ = distribution(answer)
- if label not in keys:
- raise ValueError("Label is outside the question's options.")
- rows.append({"case_id": identifier, "case_fingerprint": hashlib.sha256(canonical(case).encode()).hexdigest(), "question_id": name, "model": result["model"],
- "checkpoint": metadata["checkpoint"], "schema_hash": metadata["schema_hash"],
- "sdk_version": metadata["sdk_version"], "source_calibration": "raw",
- "answer": answer, "label": label})
- if not rows:
- raise ValueError("Empty dataset.")
- Path(output).write_text("".join(canonical(row) + "\n" for row in rows))
- return len(rows)
-
-
-def main():
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("dataset")
- parser.add_argument("--endpoint", default="http://127.0.0.1:8099/v1/decisions")
- parser.add_argument("--output", required=True)
- args = parser.parse_args()
- try:
- print(json.dumps({"observations": evaluate(args.dataset, args.endpoint, args.output)}))
- except (ValueError, KeyError, TypeError, OSError) as exc:
- parser.exit(2, f"Evaluation failed ({type(exc).__name__}); no new observations written.\n")
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/laya_service/protocol.py b/tools/laya_service/protocol.py
deleted file mode 100644
index c895a549..00000000
--- a/tools/laya_service/protocol.py
+++ /dev/null
@@ -1,108 +0,0 @@
-"""Small, dependency-free local decision protocol and artifact validation."""
-import hashlib
-import json
-import math
-import re
-from pathlib import Path
-
-SDK_VERSION = "0.3.4"
-DEFAULT_REVISION = "1c5edc17a7acd8701df6fc341c0d179f1c62c982"
-CHECKPOINTS = ("english", "multilingual", "typed-decisions")
-MODEL_FILES = ("model.safetensors", "rl_agent_config.json", "encoder/config.json",
- "tokenizer/tokenizer.json", "tokenizer/tokenizer_config.json")
-
-
-class Rejected(ValueError):
- """A safe reason code, never request text or an underlying exception message."""
-
-
-def canonical(value):
- # Preserve dictionary order: candidate order changes Laya's inputs and predictions.
- return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
-
-
-def schema_hash(questions):
- return hashlib.sha256(canonical(questions).encode()).hexdigest()
-
-
-def file_hash(path):
- digest = hashlib.sha256()
- with Path(path).open("rb") as stream:
- for chunk in iter(lambda: stream.read(1024 * 1024), b""):
- digest.update(chunk)
- return digest.hexdigest()
-
-
-def is_hex(value, length):
- return isinstance(value, str) and re.fullmatch("[0-9a-f]{%d}" % length, value) is not None
-
-
-def unique_object(pairs):
- result = {}
- for key, value in pairs:
- if key in result:
- raise Rejected("duplicate_json_key")
- result[key] = value
- return result
-
-
-def read_json(text):
- def invalid_constant(_):
- raise Rejected("nonfinite_json")
- return json.loads(text, object_pairs_hook=unique_object, parse_constant=invalid_constant)
-
-
-def validate_request(request, model):
- if not isinstance(request, dict) or set(request) - {"model", "state", "questions", "rubric_version", "language"}:
- raise Rejected("invalid_request")
- if request.get("model") != model:
- raise Rejected("model_version_mismatch")
- if not isinstance(request.get("state"), (str, dict, list)) or len(canonical(request["state"])) > 32000:
- raise Rejected("invalid_state")
- rubric = request.get("rubric_version")
- if not isinstance(rubric, str) or not re.fullmatch(r"[a-zA-Z0-9_.-]{1,80}", rubric):
- raise Rejected("invalid_rubric")
- lang = request.get("language")
- if lang is not None and (not isinstance(lang, str) or not re.fullmatch(r"[a-zA-Z0-9-]{1,35}", lang)):
- raise Rejected("invalid_language")
- questions = request.get("questions")
- if not isinstance(questions, dict) or not 1 <= len(questions) <= 16:
- raise Rejected("invalid_questions")
- for name, question in questions.items():
- if not isinstance(name, str) or not re.fullmatch(r"[a-zA-Z0-9_.-]{1,80}", name) or not isinstance(question, dict):
- raise Rejected("invalid_question")
- if set(question) - {"type", "instructions", "criteria"}:
- raise Rejected("invalid_question")
- if not isinstance(question.get("instructions"), str) or not 1 <= len(question["instructions"]) <= 2000:
- raise Rejected("invalid_instructions")
- kind, criteria = question.get("type"), question.get("criteria")
- if kind == "choice":
- if not isinstance(criteria, dict) or not 2 <= len(criteria) <= 20 or any(not isinstance(k, str) or not 1 <= len(k) <= 80 for k in criteria):
- raise Rejected("invalid_choices")
- values = criteria.values()
- elif kind == "score":
- if not isinstance(criteria, list) or not 2 <= len(criteria) <= 10:
- raise Rejected("invalid_score")
- values = criteria
- elif kind == "noul":
- if criteria is not None and (not isinstance(criteria, dict) or set(criteria) != {"false", "true"}):
- raise Rejected("invalid_noul")
- values = criteria.values() if criteria else []
- else:
- raise Rejected("invalid_question_type")
- if any(v is not None and (not isinstance(v, str) or len(v) > 2000) for v in values):
- raise Rejected("invalid_criteria")
- return questions
-
-
-def distribution(answer):
- if answer["type"] == "noul":
- probability = answer["noul"]
- values, keys = [1 - probability, probability], ["false", "true"]
- else:
- keys, values = list(answer["probabilities"]), list(answer["probabilities"].values())
- if len(values) < 2 or any(isinstance(p, bool) or not isinstance(p, (float, int)) or
- not math.isfinite(p) or not 0 <= p <= 1 for p in values) or abs(sum(values) - 1) > 0.002:
- raise Rejected("invalid_model_probabilities")
- total = sum(values)
- return keys, [p / total for p in values]
diff --git a/tools/laya_service/requirements.txt b/tools/laya_service/requirements.txt
deleted file mode 100644
index a5bf5d8e..00000000
--- a/tools/laya_service/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# Tested with Python 3.12. Install the appropriate PyTorch CUDA wheel separately on GPU hosts.
-laya==0.3.4
-torch==2.14.0
-transformers==5.17.0
-safetensors==0.8.0
-huggingface-hub==1.32.0
-numpy==2.5.3
diff --git a/tools/laya_service/runtime.py b/tools/laya_service/runtime.py
deleted file mode 100644
index 556133d9..00000000
--- a/tools/laya_service/runtime.py
+++ /dev/null
@@ -1,87 +0,0 @@
-"""Preloaded, offline Laya runtime with strict provenance and no silent truncation."""
-import json
-import os
-from pathlib import Path
-from .calibration import Calibration
-from .compat import ensure_complete, select_checkpoint
-from .protocol import (CHECKPOINTS, MODEL_FILES, SDK_VERSION, Rejected, canonical, file_hash,
- is_hex, schema_hash, validate_request)
-
-
-def load_manifest(path):
- path = Path(path).resolve()
- data = json.loads(path.read_text())
- if data.get("version") != 1 or not is_hex(data.get("revision"), 40) or not isinstance(data.get("checkpoints"), dict) or not data["checkpoints"]:
- raise ValueError("Invalid checkpoint manifest.")
- for name, checkpoint in data["checkpoints"].items():
- if name not in CHECKPOINTS or not isinstance(checkpoint, dict):
- raise ValueError("Invalid checkpoint entry.")
- directory = (path.parent / checkpoint["path"]).resolve()
- if not directory.is_relative_to(path.parent):
- raise ValueError("Checkpoint path must remain inside the manifest directory.")
- if set(checkpoint["sha256"]) != set(MODEL_FILES):
- raise ValueError("Incomplete checkpoint manifest.")
- for filename, expected in checkpoint["sha256"].items():
- target = (directory / filename).resolve()
- if not target.is_relative_to(path.parent) or not target.is_file() or not is_hex(expected, 64) or file_hash(target) != expected:
- raise ValueError("Checkpoint file missing or hash mismatch.")
- checkpoint["resolved_path"] = str(directory)
- return data
-
-
-class Runtime:
- def __init__(self, manifest, calibration=None, device="cpu", checkpoint="auto", threads=4):
- # Set before importing ML libraries. All required assets were validated locally.
- os.environ.update(HF_HUB_OFFLINE="1", TRANSFORMERS_OFFLINE="1", USE_TF="0", TOKENIZERS_PARALLELISM="false")
- import laya
- import torch
- if laya.__version__ != SDK_VERSION:
- raise ValueError("Unsupported Laya SDK version; install the pinned requirements.")
- data = load_manifest(manifest)
- self.model = "laya@" + data["revision"]
- self.revision = data["revision"]
- self.calibration = Calibration(calibration, self.model)
- self.checkpoint = checkpoint
- if checkpoint != "auto" and checkpoint not in data["checkpoints"]:
- raise ValueError("Requested checkpoint is not installed.")
- torch.set_num_threads(threads)
- names = data["checkpoints"] if checkpoint == "auto" else [checkpoint]
- self.agents = {}
- warmup = {"ready": {"type": "choice", "instructions": "What is the message?",
- "criteria": {"greeting": "a greeting", "other": "anything else"}}}
- for name in names:
- agent = laya.load(data["checkpoints"][name]["resolved_path"], device=device)
- ensure_complete(agent, "Hello", warmup)
- agent.predict("Hello", warmup)
- self.agents[name] = agent
-
- def health(self):
- return {"ready": True, "model": self.model, "calibration_id": self.calibration.identifier,
- "sdk_version": SDK_VERSION, "checkpoints": {name: str(agent.device) for name, agent in self.agents.items()}}
-
- def predict(self, request):
- questions = validate_request(request, self.model)
- selected = select_checkpoint(request["state"], request.get("language"))
- if self.checkpoint != "auto":
- if self.checkpoint in ("english", "typed-decisions") and selected != "english":
- raise Rejected("language_checkpoint_conflict")
- selected = self.checkpoint
- if selected not in self.agents:
- raise Rejected("checkpoint_not_installed")
- agent = self.agents[selected]
- schema = schema_hash(questions)
- self.calibration.check(schema, selected, questions)
- ensure_complete(agent, request["state"], questions)
- result = agent.predict(request["state"], questions)
- try:
- raw = result["answers"]
- response = {"model": self.model, "answers": self.calibration.apply(selected, raw), "raw_answers": raw,
- "usage": result["usage"],
- "metadata": {"checkpoint": selected, "revision": self.revision,
- "calibration_id": self.calibration.identifier, "schema_hash": schema,
- "rubric_version": request["rubric_version"], "device": agent.device.type,
- "sdk_version": SDK_VERSION, "truncated": False}}
- canonical(response)
- return response
- except (Rejected, ValueError, TypeError, KeyError, RecursionError) as exc:
- raise RuntimeError("invalid_model_output") from exc
diff --git a/tools/laya_service/tests/CalibrationFitterTests.cs b/tools/laya_service/tests/CalibrationFitterTests.cs
new file mode 100644
index 00000000..4ac360a2
--- /dev/null
+++ b/tools/laya_service/tests/CalibrationFitterTests.cs
@@ -0,0 +1,159 @@
+using System.Text.Json;
+using OpenClaw.LayaService.Evaluation;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class CalibrationFitterTests
+{
+ [Fact]
+ public void Metrics_MeasuresAccuracyNllBrierEceAndRiskCoverage()
+ {
+ var rows = new[]
+ {
+ new LabeledPrediction("choice", ["a", "b"], [0.8, 0.2], "a"),
+ new LabeledPrediction("choice", ["a", "b"], [0.3, 0.7], "a")
+ };
+
+ var metrics = CalibrationMetrics.Measure(rows);
+
+ Assert.Equal(2, metrics.Samples);
+ Assert.Equal(0.5, metrics.Accuracy);
+ Assert.Equal((-Math.Log(0.8) - Math.Log(0.3)) / 2, metrics.Nll, 10);
+ Assert.Equal((0.08 + 0.98) / 2, metrics.Brier, 10);
+ Assert.Equal(2, metrics.ReliabilityBins.Count);
+ Assert.Equal(8, metrics.RiskCoverage.Count);
+ }
+
+ [Fact]
+ public void Metrics_NormalizesRoundedProbabilitiesAndSupportsNoulAndScore()
+ {
+ var rows = new[]
+ {
+ new LabeledPrediction("noul", ["false", "true"], [0.2495, 0.7495], "true"),
+ new LabeledPrediction("score", ["0", "1"], [0.8, 0.2], "1")
+ };
+
+ var metrics = CalibrationMetrics.Measure(rows);
+
+ Assert.Equal(2, metrics.Samples);
+ Assert.Equal(0.5, metrics.Accuracy);
+ Assert.Equal((-Math.Log(0.7495 / 0.999) - Math.Log(0.2)) / 2, metrics.Nll, 10);
+ }
+
+ [Fact]
+ public void Fit_ProducesVersionTwoArtifactFromDisjointRawObservations()
+ {
+ var training = BuildObservations("train", 20);
+ var validation = BuildObservations("validation", 20);
+
+ var artifact = CalibrationFitter.Fit(training, validation);
+ var json = CalibrationFitter.Serialize(artifact);
+
+ using var document = JsonDocument.Parse(json);
+ Assert.Equal(2, document.RootElement.GetProperty("version").GetInt32());
+ Assert.Equal("1.0.0", document.RootElement.GetProperty("sdk_version").GetString());
+ Assert.Equal("NLaya", document.RootElement.GetProperty("runtime").GetString());
+ Assert.Equal(0.1, document.RootElement.GetProperty("temperatures").GetProperty("english").GetProperty("choice:2").GetDouble());
+ Assert.EndsWith("\n", json, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public void Fit_RejectsMixedModelSchemaRuntimeAndInsufficientBucketSamples()
+ {
+ var training = BuildObservations("train", 20);
+ var validation = BuildObservations("validation", 20);
+ Assert.Throws(() => CalibrationFitter.Fit(training, [training[0] with { Model = "laya@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }]));
+
+ var mixedSchema = validation.ToArray();
+ mixedSchema[0] = mixedSchema[0] with { SchemaHash = new string('b', 64) };
+ Assert.Throws(() => CalibrationFitter.Fit(training, mixedSchema));
+
+ var mixedRuntime = validation.ToArray();
+ mixedRuntime[0] = mixedRuntime[0] with { Runtime = "OtherRuntime" };
+ Assert.Throws(() => CalibrationFitter.Fit(training, mixedRuntime));
+ var unknownCheckpoint = validation.ToArray();
+ unknownCheckpoint[0] = unknownCheckpoint[0] with { Checkpoint = "unknown" };
+ Assert.Throws(() => CalibrationFitter.Fit(training, unknownCheckpoint));
+ Assert.Throws(() => CalibrationFitter.Fit(training[..19], validation));
+ }
+
+ [Fact]
+ public void Fit_RejectsMismatchedBuckets()
+ {
+ var training = BuildObservations("train", 20);
+ var validation = BuildObservations("validation", 20);
+ using var answer = JsonDocument.Parse("""{"type":"score","score":0,"probabilities":{"0":0.7,"1":0.3}}""");
+ validation[0] = validation[0] with { RawAnswer = answer.RootElement.Clone() };
+
+ Assert.Throws(() => CalibrationFitter.Fit(training, validation));
+ }
+
+ [Fact]
+ public void Fit_RejectsNonNumericRawProbabilities()
+ {
+ var training = BuildObservations("train", 20);
+ var validation = BuildObservations("validation", 20);
+ using var answer = JsonDocument.Parse("""{"type":"choice","probabilities":{"a":"0.6","b":0.4}}""");
+ training[0] = training[0] with { RawAnswer = answer.RootElement.Clone() };
+
+ Assert.Throws(() => CalibrationFitter.Fit(training, validation));
+ }
+
+ [Fact]
+ public async Task WriteAsync_ReturnsSha256OfExactArtifactBytes()
+ {
+ var artifact = CalibrationFitter.Fit(BuildObservations("train", 20), BuildObservations("validation", 20));
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"), "calibration.json");
+ try
+ {
+ var identifier = await CalibrationFitter.WriteAsync(artifact, path, CancellationToken.None);
+ var payload = await File.ReadAllBytesAsync(path);
+
+ Assert.Equal(Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(payload)), identifier);
+ Assert.Equal((byte)'\n', payload[^1]);
+ }
+ finally
+ {
+ var directory = Path.GetDirectoryName(path)!;
+ if (Directory.Exists(directory)) Directory.Delete(directory, recursive: true);
+ }
+ }
+
+ [Fact]
+ public void Fit_RejectsOverlappingCasesAndFingerprints()
+ {
+ var training = BuildObservations("train", 20);
+ var validation = BuildObservations("validation", 20).ToArray();
+ validation[0] = validation[0] with
+ {
+ CaseId = training[0].CaseId,
+ CaseFingerprint = training[0].CaseFingerprint
+ };
+
+ Assert.Throws(() => CalibrationFitter.Fit(training, validation));
+ }
+
+ private static Observation[] BuildObservations(string prefix, int count)
+ {
+ var rows = new Observation[count];
+ for (var index = 0; index < count; index++)
+ {
+ var correct = index % 2 == 0;
+ using var answer = JsonDocument.Parse(correct
+ ? """{"type":"choice","choice":"a","confidence":0.3,"answer_confidence":0.6,"probabilities":{"a":0.6,"b":0.4}}"""
+ : """{"type":"choice","choice":"b","confidence":0.3,"answer_confidence":0.6,"probabilities":{"a":0.4,"b":0.6}}""");
+ rows[index] = new Observation(
+ prefix + "-" + index,
+ Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(prefix + index))),
+ "tier",
+ "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982",
+ "english",
+ new string('a', 64),
+ "1.0.0",
+ answer.RootElement.Clone(),
+ correct ? "a" : "b");
+ }
+ return rows;
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/CaseEvaluatorTests.cs b/tools/laya_service/tests/CaseEvaluatorTests.cs
new file mode 100644
index 00000000..bdb17956
--- /dev/null
+++ b/tools/laya_service/tests/CaseEvaluatorTests.cs
@@ -0,0 +1,162 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using OpenClaw.LayaService.Evaluation;
+using OpenClaw.LayaService.Protocol;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class CaseEvaluatorTests
+{
+ private const string Model = "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982";
+
+ [Fact]
+ public async Task EvaluateAsync_UsesDefaultsAndWritesStateFreeObservations()
+ {
+ var directory = CreateTempDirectory();
+ var dataset = Path.Combine(directory, "cases.jsonl");
+ var output = Path.Combine(directory, "observations.jsonl");
+ await File.WriteAllTextAsync(dataset, """
+ {"case_id":"case-one","state":"private state text","labels":{"tier":"T0","high_risk":false,"requires_tools":true}}
+ """ + "\n");
+ var handler = new RespondingHandler();
+ using var http = new HttpClient(handler);
+ try
+ {
+ var count = await CaseEvaluator.EvaluateAsync(dataset, new Uri("http://127.0.0.1:8765/v1/decisions"), output, http, CancellationToken.None);
+
+ Assert.Equal(3, count);
+ Assert.Equal(Model, handler.Request!.Model);
+ Assert.Equal("openclaw-laya-tiers-v1", handler.Request.RubricVersion);
+ var observations = await File.ReadAllLinesAsync(output);
+ Assert.Equal(3, observations.Length);
+ foreach (var line in observations)
+ {
+ using var observation = JsonDocument.Parse(line);
+ Assert.False(observation.RootElement.TryGetProperty("state", out _));
+ Assert.Equal("raw", observation.RootElement.GetProperty("source_calibration").GetString());
+ Assert.Equal("NLaya", observation.RootElement.GetProperty("runtime").GetString());
+ Assert.Equal("1.0.0", observation.RootElement.GetProperty("sdk_version").GetString());
+ Assert.Equal(64, observation.RootElement.GetProperty("case_fingerprint").GetString()!.Length);
+ Assert.False(observation.RootElement.GetProperty("answer").ValueKind == JsonValueKind.Undefined);
+ }
+ Assert.Equal(1, handler.Calls);
+ }
+ finally
+ {
+ Directory.Delete(directory, recursive: true);
+ }
+ }
+
+ [Fact]
+ public async Task EvaluateAsync_RejectsRemoteEndpointBeforeSending()
+ {
+ var directory = CreateTempDirectory();
+ var dataset = Path.Combine(directory, "cases.jsonl");
+ await File.WriteAllTextAsync(dataset, "{}");
+ var handler = new RespondingHandler();
+ using var http = new HttpClient(handler);
+ try
+ {
+ await Assert.ThrowsAsync(() => CaseEvaluator.EvaluateAsync(
+ dataset, new Uri("https://example.com/v1/decisions"), Path.Combine(directory, "out.jsonl"), http, CancellationToken.None));
+ Assert.Equal(0, handler.Calls);
+ }
+ finally
+ {
+ Directory.Delete(directory, recursive: true);
+ }
+ }
+
+ [Fact]
+ public async Task EvaluateAsync_MetadataFailureDoesNotReplacePreviousOutput()
+ {
+ var directory = CreateTempDirectory();
+ var dataset = Path.Combine(directory, "cases.jsonl");
+ var output = Path.Combine(directory, "observations.jsonl");
+ await File.WriteAllTextAsync(dataset, """
+ {"case_id":"case-one","model":"laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982","state":"hello","rubric_version":"v1","questions":{"tier":{"type":"choice","instructions":"Tier?","criteria":{"T0":"small","T1":"large"}}},"labels":{"tier":"T0"}}
+ """ + "\n");
+ await File.WriteAllTextAsync(output, "previous observations\n");
+ var handler = new RespondingHandler { Runtime = "wrong" };
+ using var http = new HttpClient(handler);
+ try
+ {
+ await Assert.ThrowsAsync(() => CaseEvaluator.EvaluateAsync(
+ dataset, new Uri("http://127.0.0.1:8765/v1/decisions"), output, http, CancellationToken.None));
+ Assert.Equal("previous observations\n", await File.ReadAllTextAsync(output));
+ Assert.Empty(Directory.EnumerateFiles(directory, "*.tmp", SearchOption.TopDirectoryOnly));
+ }
+ finally
+ {
+ Directory.Delete(directory, recursive: true);
+ }
+ }
+
+ private static string CreateTempDirectory()
+ {
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(path);
+ return path;
+ }
+
+ private sealed class RespondingHandler : HttpMessageHandler
+ {
+ public int Calls { get; private set; }
+ public string Runtime { get; init; } = "NLaya";
+ public DecisionWireRequest? Request { get; private set; }
+
+ protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+ {
+ Calls++;
+ using var body = JsonDocument.Parse(await request.Content!.ReadAsByteArrayAsync(cancellationToken));
+ Request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(body.RootElement.GetRawText()));
+ var answers = new JsonObject();
+ foreach (var question in Request.Questions.EnumerateObject())
+ {
+ if (question.Value.GetProperty("type").GetString() == "choice")
+ {
+ var probabilities = new JsonObject();
+ foreach (var candidate in question.Value.GetProperty("criteria").EnumerateObject())
+ probabilities[candidate.Name] = candidate.Name == "T0" ? 1 : 0;
+ answers[question.Name] = new JsonObject
+ {
+ ["type"] = "choice", ["choice"] = probabilities.First().Key,
+ ["confidence"] = 1, ["answer_confidence"] = 1, ["probabilities"] = probabilities
+ };
+ }
+ else
+ {
+ answers[question.Name] = new JsonObject
+ {
+ ["type"] = "noul", ["noul"] = 0.8, ["value"] = true,
+ ["confidence"] = 0.8, ["answer_confidence"] = 0.8
+ };
+ }
+ }
+
+ var response = new JsonObject
+ {
+ ["model"] = Request.Model,
+ ["answers"] = answers,
+ ["raw_answers"] = answers.DeepClone(),
+ ["usage"] = new JsonObject { ["input_tokens"] = 8, ["output_tokens"] = 0 },
+ ["metadata"] = new JsonObject
+ {
+ ["checkpoint"] = "english", ["revision"] = Request.Model[5..],
+ ["calibration_id"] = "uncalibrated", ["schema_hash"] = StrictJson.SchemaHash(Request.Questions),
+ ["rubric_version"] = Request.RubricVersion, ["device"] = "cpu", ["sdk_version"] = "1.0.0",
+ ["runtime"] = Runtime, ["truncated"] = false
+ }
+ };
+ return new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ RequestMessage = request,
+ Content = new StringContent(response.ToJsonString())
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/CommandLineTests.cs b/tools/laya_service/tests/CommandLineTests.cs
new file mode 100644
index 00000000..b060b330
--- /dev/null
+++ b/tools/laya_service/tests/CommandLineTests.cs
@@ -0,0 +1,183 @@
+using OpenClaw.LayaService;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class CommandLineTests
+{
+ [Theory]
+ [InlineData("serve --manifest models.json", "serve")]
+ [InlineData("download --destination models", "download")]
+ [InlineData("evaluate cases.jsonl --output observations.jsonl", "evaluate")]
+ [InlineData("calibrate --fit train.jsonl --validate holdout.jsonl --output calibration.json", "calibrate")]
+ [InlineData("report journal.jsonl", "report")]
+ public void Parse_AcceptsAllCommands(string args, string expectedCommand)
+ {
+ var parsed = CommandLine.Parse(args.Split(' '));
+
+ Assert.Equal(expectedCommand, parsed.Command);
+ }
+
+ [Fact]
+ public void Parse_DownloadPreservesRepeatedCheckpoints()
+ {
+ var parsed = CommandLine.Parse(["download", "--destination", "models", "--checkpoint", "english", "--checkpoint", "multilingual"]);
+
+ Assert.Equal("download", parsed.Command);
+ Assert.Equal(new[] { "english", "multilingual" }, parsed.Options.GetMany("checkpoint"));
+ }
+
+ [Fact]
+ public void Parse_PreservesPositionalArguments()
+ {
+ var parsed = CommandLine.Parse(["evaluate", "cases.jsonl", "--output", "observations.jsonl"]);
+
+ Assert.Equal(new[] { "cases.jsonl" }, parsed.Arguments);
+ }
+
+ [Fact]
+ public async Task Download_InvalidRevisionFailsBeforeAnyNetworkRequest()
+ {
+ var directoryPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(directoryPath);
+ var previousError = Console.Error;
+ using var error = new StringWriter();
+ try
+ {
+ Console.SetError(error);
+ var exitCode = await Program.Main(["download", "--destination", directoryPath,
+ "--revision", "latest", "--checkpoint", "english"]);
+
+ Assert.Equal(2, exitCode);
+ Assert.Equal("invalid_arguments", error.ToString().Trim());
+ Assert.Empty(Directory.EnumerateFileSystemEntries(directoryPath));
+ }
+ finally
+ {
+ Console.SetError(previousError);
+ Directory.Delete(directoryPath, recursive: true);
+ }
+ }
+
+ [Theory]
+ [InlineData("multilingual", "checkpoint_not_installed")]
+ [InlineData("unknown", "unknown_checkpoint")]
+ public async Task Serve_ReportsCheckpointStartupFailure(string checkpoint, string expectedReason)
+ {
+ var directoryPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(directoryPath);
+ var manifestPath = ModelManifestTests.WriteManifest(directoryPath, new string('a', 40), ["english"]);
+ var previousError = Console.Error;
+ using var error = new StringWriter();
+ try
+ {
+ Console.SetError(error);
+ var exitCode = await Program.Main(["serve", "--manifest", manifestPath, "--checkpoint", checkpoint]);
+
+ Assert.Equal(2, exitCode);
+ Assert.Equal(expectedReason, error.ToString().Trim());
+ }
+ finally
+ {
+ Console.SetError(previousError);
+ Directory.Delete(directoryPath, recursive: true);
+ }
+ }
+
+ [Fact]
+ public async Task Calibrate_FitsObservationFilesAndWritesV2Artifact()
+ {
+ var directoryPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(directoryPath);
+ var trainingPath = Path.Combine(directoryPath, "training.jsonl");
+ var validationPath = Path.Combine(directoryPath, "validation.jsonl");
+ var outputPath = Path.Combine(directoryPath, "calibration.json");
+ await File.WriteAllLinesAsync(trainingPath, BuildObservations("train", 20));
+ await File.WriteAllLinesAsync(validationPath, BuildObservations("validation", 20));
+ var previousOutput = Console.Out;
+ using var output = new StringWriter();
+ try
+ {
+ Console.SetOut(output);
+ var exitCode = await Program.Main(["calibrate", "--fit", trainingPath, "--validate", validationPath, "--output", outputPath]);
+
+ Assert.Equal(0, exitCode);
+ using var artifact = JsonDocument.Parse(await File.ReadAllBytesAsync(outputPath));
+ Assert.Equal(2, artifact.RootElement.GetProperty("version").GetInt32());
+ Assert.Equal("NLaya", artifact.RootElement.GetProperty("runtime").GetString());
+ var payload = await File.ReadAllBytesAsync(outputPath);
+ Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(payload)), output.ToString().Trim());
+ }
+ finally
+ {
+ Console.SetOut(previousOutput);
+ Directory.Delete(directoryPath, recursive: true);
+ }
+ }
+
+ [Fact]
+ public async Task Report_WritesSummaryJsonAndReliabilityPlot()
+ {
+ var directoryPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(directoryPath);
+ var journalPath = Path.Combine(directoryPath, "journal.jsonl");
+ var labelsPath = Path.Combine(directoryPath, "labels.jsonl");
+ var outputPath = Path.Combine(directoryPath, "report.json");
+ var plotPath = Path.Combine(directoryPath, "report.png");
+ await File.WriteAllTextAsync(journalPath,
+ """{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","proposed_tier":"T0","latency_ms":10,"input_tokens":100,"estimated_cost_usd":0.000001,"probabilities":{"T0":0.8,"T1":0.1,"T2":0.05,"T3":0.05}}""" + "\n");
+ await File.WriteAllTextAsync(labelsPath, """{"decision_id":"a","expected_tier":"T0"}""" + "\n");
+ try
+ {
+ var exitCode = await Program.Main(["report", journalPath, "--labels", labelsPath, "--output", outputPath, "--plot", plotPath]);
+
+ Assert.Equal(0, exitCode);
+ using var report = JsonDocument.Parse(await File.ReadAllBytesAsync(outputPath));
+ Assert.Equal(1, report.RootElement.GetProperty("decisions").GetInt32());
+ Assert.Equal(1, report.RootElement.GetProperty("quality").GetProperty("jev_with_fallback").GetProperty("accuracy").GetDouble());
+ Assert.True(new FileInfo(plotPath).Length > 100);
+ }
+ finally
+ {
+ Directory.Delete(directoryPath, recursive: true);
+ }
+ }
+
+ private static string[] BuildObservations(string prefix, int count)
+ {
+ var rows = new string[count];
+ for (var index = 0; index < count; index++)
+ {
+ var id = prefix + "-" + index;
+ var fingerprint = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(id)));
+ var answer = index % 2 == 0
+ ? "{\"type\":\"choice\",\"choice\":\"a\",\"probabilities\":{\"a\":0.7,\"b\":0.3}}"
+ : "{\"type\":\"choice\",\"choice\":\"b\",\"probabilities\":{\"a\":0.3,\"b\":0.7}}";
+ rows[index] = JsonSerializer.Serialize(new
+ {
+ case_id = id,
+ case_fingerprint = fingerprint,
+ question_id = "tier",
+ model = "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982",
+ checkpoint = "english",
+ schema_hash = new string('a', 64),
+ sdk_version = "1.0.0",
+ runtime = "NLaya",
+ source_calibration = "raw",
+ answer = JsonDocument.Parse(answer).RootElement,
+ label = index % 2 == 0 ? "a" : "b"
+ });
+ }
+ return rows;
+ }
+
+ [Theory]
+ [InlineData("unknown")]
+ [InlineData("download --unknown value")]
+ [InlineData("download --destination")]
+ public void Parse_RejectsUnknownCommandsAndMalformedOptions(string args)
+ => Assert.Throws(() => CommandLine.Parse(args.Split(' ')));
+}
diff --git a/tools/laya_service/tests/DecisionServerTests.cs b/tools/laya_service/tests/DecisionServerTests.cs
new file mode 100644
index 00000000..d7d94e32
--- /dev/null
+++ b/tools/laya_service/tests/DecisionServerTests.cs
@@ -0,0 +1,148 @@
+using System.Net;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting.Server;
+using Microsoft.AspNetCore.Hosting.Server.Features;
+using Microsoft.Extensions.DependencyInjection;
+using OpenClaw.LayaService.Hosting;
+using OpenClaw.LayaService.Protocol;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class DecisionServerTests : IAsyncLifetime
+{
+ private const string Model = "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982";
+ private const string GoldenRequest = "{\"model\":\"" + Model + "\",\"state\":\"hello\",\"questions\":{\"tier\":{\"type\":\"choice\",\"instructions\":\"Which task?\",\"criteria\":{\"small\":\"simple\",\"large\":\"complex\"}}},\"rubric_version\":\"test-v1\"}";
+ private readonly FakePredictor _predictor = new();
+ private WebApplication _app = null!;
+ private HttpClient _http = null!;
+
+ public async ValueTask InitializeAsync()
+ {
+ _app = DecisionServer.Build(_predictor, new ServiceOptions(0, 8, 65536, TimeSpan.FromSeconds(5)));
+ await _app.StartAsync();
+ var server = _app.Services.GetRequiredService();
+ var address = server.Features.Get()!.Addresses.Single();
+ _http = new HttpClient { BaseAddress = new Uri(address) };
+ }
+
+ public async ValueTask DisposeAsync()
+ {
+ _http.Dispose();
+ await _app.StopAsync();
+ await _app.DisposeAsync();
+ }
+
+ [Fact]
+ public async Task HealthAndValidRequest_ReturnPredictorResponses()
+ {
+ using var health = await _http.GetAsync("/health");
+ Assert.Equal(HttpStatusCode.OK, health.StatusCode);
+ Assert.True(JsonDocument.Parse(await health.Content.ReadAsStringAsync()).RootElement.GetProperty("ready").GetBoolean());
+
+ using var response = await _http.SendAsync(CreatePost());
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ Assert.True(JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement.GetProperty("ok").GetBoolean());
+ Assert.Equal(1, _predictor.Calls);
+ }
+
+ [Fact]
+ public async Task RejectsOriginHostChunkedAndOversizedBodies()
+ {
+ using var originRequest = CreatePost();
+ originRequest.Headers.Add("Origin", "https://example.invalid");
+ using var originResponse = await _http.SendAsync(originRequest);
+ Assert.Equal(HttpStatusCode.Forbidden, originResponse.StatusCode);
+
+ using var hostRequest = CreatePost();
+ hostRequest.Headers.Host = "attacker.invalid";
+ using var hostResponse = await _http.SendAsync(hostRequest);
+ Assert.Equal(HttpStatusCode.Forbidden, hostResponse.StatusCode);
+
+ using var chunkedRequest = CreatePost();
+ chunkedRequest.Headers.TransferEncodingChunked = true;
+ using var chunkedResponse = await _http.SendAsync(chunkedRequest);
+ Assert.Equal(HttpStatusCode.UnsupportedMediaType, chunkedResponse.StatusCode);
+
+ using var largeRequest = new HttpRequestMessage(HttpMethod.Post, "/v1/decisions")
+ {
+ Content = new StringContent(new string('x', 65537), Encoding.UTF8, "application/json")
+ };
+ using var largeResponse = await _http.SendAsync(largeRequest);
+ Assert.Equal(HttpStatusCode.RequestEntityTooLarge, largeResponse.StatusCode);
+ Assert.Equal(0, _predictor.Calls);
+ }
+
+ [Fact]
+ public async Task InvalidRequestDoesNotEchoInput()
+ {
+ var secretRequest = GoldenRequest.Replace(Model, "SECRET", StringComparison.Ordinal);
+ using var response = await _http.PostAsync("/v1/decisions", JsonContent(secretRequest));
+ var body = await response.Content.ReadAsStringAsync();
+
+ Assert.Equal(HttpStatusCode.UnprocessableEntity, response.StatusCode);
+ Assert.DoesNotContain("SECRET", body, StringComparison.Ordinal);
+ Assert.Equal("model_version_mismatch", JsonDocument.Parse(body).RootElement.GetProperty("error").GetString());
+ Assert.Equal(0, _predictor.Calls);
+ }
+
+ [Fact]
+ public async Task PredictorFailureReturnsFixedServiceError()
+ {
+ _predictor.Failure = new InvalidOperationException("sensitive inference detail");
+ using var response = await _http.SendAsync(CreatePost());
+
+ Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+ Assert.Equal("{\"error\":\"inference_failed\"}", await response.Content.ReadAsStringAsync());
+ }
+
+ [Fact]
+ public async Task ConcurrentInferenceIsRejectedWithoutQueueing()
+ {
+ _predictor.Block = true;
+ using var firstRequest = CreatePost();
+ var firstResponseTask = _http.SendAsync(firstRequest);
+ await _predictor.Entered.Task.WaitAsync(TimeSpan.FromSeconds(3));
+
+ using var secondResponse = await _http.SendAsync(CreatePost());
+ Assert.Equal(HttpStatusCode.ServiceUnavailable, secondResponse.StatusCode);
+ Assert.Equal("busy", JsonDocument.Parse(await secondResponse.Content.ReadAsStringAsync()).RootElement.GetProperty("error").GetString());
+
+ _predictor.Release.TrySetResult();
+ using var firstResponse = await firstResponseTask;
+ Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode);
+ }
+
+ private static HttpRequestMessage CreatePost()
+ => new(HttpMethod.Post, "/v1/decisions") { Content = JsonContent(GoldenRequest) };
+
+ private static StringContent JsonContent(string value)
+ => new(value, Encoding.UTF8, "application/json");
+
+ private sealed class FakePredictor : IDecisionPredictor
+ {
+ public string Model => DecisionServerTests.Model;
+ private static readonly JsonElement Health = JsonDocument.Parse("{\"ready\":true}").RootElement.Clone();
+ private static readonly JsonElement Result = JsonDocument.Parse("{\"ok\":true}").RootElement.Clone();
+
+ public int Calls { get; private set; }
+ public Exception? Failure { get; set; }
+ public bool Block { get; set; }
+ public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ public TaskCompletionSource Release { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+ public JsonElement GetHealth() => Health;
+
+ public async Task PredictAsync(DecisionWireRequest request, CancellationToken cancellationToken)
+ {
+ Calls++;
+ Entered.TrySetResult();
+ if (Block) await Release.Task.WaitAsync(cancellationToken);
+ if (Failure is not null) throw Failure;
+ return Result;
+ }
+ }
+}
diff --git a/tools/laya_service/tests/HuggingFaceDownloaderTests.cs b/tools/laya_service/tests/HuggingFaceDownloaderTests.cs
new file mode 100644
index 00000000..29b188e5
--- /dev/null
+++ b/tools/laya_service/tests/HuggingFaceDownloaderTests.cs
@@ -0,0 +1,290 @@
+using System.Net;
+using System.Security.Cryptography;
+using System.Text;
+using OpenClaw.LayaService.Models;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class HuggingFaceDownloaderTests
+{
+ private const string Revision = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+ [Fact]
+ public async Task DownloadAsync_FetchesAllowlistedFilesWritesVerifiedManifestAndNotices()
+ {
+ using var directory = new TemporaryDirectory();
+ var handler = new FakeHandler((request, _) => Task.FromResult(Success(request)));
+ using var http = new HttpClient(handler);
+
+ var manifestPath = await HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english", "multilingual", "typed-decisions"]), http, CancellationToken.None);
+
+ Assert.Equal(Path.Combine(directory.Path, Revision, "manifest.json"), manifestPath);
+ Assert.Equal(15, handler.Requests.Count);
+ Assert.All(handler.Requests, request =>
+ {
+ Assert.Equal("huggingface.co", request.Host);
+ Assert.StartsWith($"/convaiinnovations/laya/resolve/{Revision}/", request.AbsolutePath, StringComparison.Ordinal);
+ });
+ Assert.True(File.Exists(Path.Combine(directory.Path, Revision, "LAYA-NOTICE.md")));
+ Assert.True(File.Exists(Path.Combine(directory.Path, Revision, "LAYA-LICENSE.txt")));
+
+ var verified = ModelManifest.LoadAndVerify(manifestPath, Revision);
+ Assert.Equal(3, verified.Checkpoints.Count);
+ var normalizedTokenizer = File.ReadAllText(Path.Combine(directory.Path, Revision, "hub", "tokenizer", "tokenizer_config.json"));
+ Assert.Contains("PreTrainedTokenizerFast", normalizedTokenizer, StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task DownloadAsync_RejectsInvalidRevisionCheckpointHttpErrorsAndRedirects()
+ {
+ using var directory = new TemporaryDirectory();
+ using var successfulHttp = new HttpClient(new FakeHandler((request, _) => Task.FromResult(Success(request))));
+
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, "latest", ["english"]), successfulHttp, CancellationToken.None));
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["unknown"]), successfulHttp, CancellationToken.None));
+
+ foreach (var statusCode in new[] { HttpStatusCode.NotFound, HttpStatusCode.Redirect })
+ {
+ using var failingHttp = new HttpClient(new FakeHandler((request, _) => Task.FromResult(new HttpResponseMessage(statusCode)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri("https://example.invalid/redirect") }
+ })));
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), failingHttp, CancellationToken.None));
+ Assert.False(File.Exists(Path.Combine(directory.Path, Revision, "manifest.json")));
+ }
+ }
+
+ [Fact]
+ public async Task DownloadAsync_FollowsSingleRedirectToOfficialHuggingFaceCdn()
+ {
+ using var directory = new TemporaryDirectory();
+ var handler = new FakeHandler((request, _) =>
+ {
+ if (request.RequestUri!.Host == "huggingface.co")
+ {
+ var cdnUri = new Uri("https://us.aws.cdn.hf.co" + request.RequestUri.PathAndQuery + "?signed=test");
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.Found)
+ {
+ RequestMessage = request,
+ Headers = { Location = cdnUri }
+ });
+ }
+
+ return Task.FromResult(Success(request));
+ });
+ using var http = new HttpClient(handler);
+
+ var manifestPath = await HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), http, CancellationToken.None);
+
+ Assert.Equal(10, handler.Requests.Count);
+ Assert.Equal(5, handler.Requests.Count(request => request.Host == "huggingface.co"));
+ Assert.Equal(5, handler.Requests.Count(request => request.Host.EndsWith(".cdn.hf.co", StringComparison.Ordinal)));
+ Assert.Single(ModelManifest.LoadAndVerify(manifestPath, Revision).Checkpoints);
+ }
+
+ [Fact]
+ public async Task DownloadAsync_FollowsPinnedResolveCacheThenOfficialCdn()
+ {
+ using var directory = new TemporaryDirectory();
+ var handler = new FakeHandler((request, _) =>
+ {
+ var file = GetModelFile(request.RequestUri!);
+ if (request.RequestUri!.Host == "huggingface.co" && !request.RequestUri.AbsolutePath.Contains("/api/resolve-cache/", StringComparison.Ordinal))
+ {
+ var cachePath = $"/api/resolve-cache/models/convaiinnovations/laya/{Revision}/{Uri.EscapeDataString(GetSourceFile(request.RequestUri))}";
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.TemporaryRedirect)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri(cachePath + "?etag=fixture", UriKind.Relative) }
+ });
+ }
+
+ if (request.RequestUri.Host == "huggingface.co" && file == "model.safetensors")
+ {
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.Found)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri($"https://us.aws.cdn.hf.co/{file}?signed=fixture") }
+ });
+ }
+
+ return Task.FromResult(SuccessForFile(request, file));
+ });
+ using var http = new HttpClient(handler);
+
+ var manifestPath = await HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), http, CancellationToken.None);
+
+ Assert.Equal(11, handler.Requests.Count);
+ Assert.Equal(5, handler.Requests.Count(request => request.Host == "huggingface.co" && !request.AbsolutePath.Contains("/api/resolve-cache/", StringComparison.Ordinal)));
+ Assert.Equal(5, handler.Requests.Count(request => request.Host == "huggingface.co" && request.AbsolutePath.Contains("/api/resolve-cache/", StringComparison.Ordinal)));
+ Assert.Single(handler.Requests, request => request.Host.EndsWith(".cdn.hf.co", StringComparison.Ordinal));
+ Assert.Single(ModelManifest.LoadAndVerify(manifestPath, Revision).Checkpoints);
+ }
+
+ [Fact]
+ public async Task DownloadAsync_RejectsUntrustedCdnLookalikesAndAdditionalRedirects()
+ {
+ foreach (var scenario in new[] { "lookalike", "second-hop" })
+ {
+ using var directory = new TemporaryDirectory();
+ using var http = new HttpClient(new FakeHandler((request, _) =>
+ {
+ if (request.RequestUri!.Host == "huggingface.co")
+ {
+ var host = scenario == "lookalike" ? "us.aws.cdn.hf.co.attacker.invalid" : "us.aws.cdn.hf.co";
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.Found)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri($"https://{host}{request.RequestUri.PathAndQuery}") }
+ });
+ }
+
+ return Task.FromResult(new HttpResponseMessage(HttpStatusCode.Found)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri("https://us.aws.cdn.hf.co/another-hop") }
+ });
+ }));
+
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), http, CancellationToken.None));
+ Assert.False(File.Exists(Path.Combine(directory.Path, Revision, "manifest.json")));
+ }
+ }
+
+ [Fact]
+ public async Task DownloadAsync_RejectsResolveCachePathForAnotherRevisionOrFile()
+ {
+ foreach (var cachePath in new[]
+ {
+ $"/api/resolve-cache/models/convaiinnovations/laya/{new string('b', 40)}/model.safetensors?etag=test",
+ $"/api/resolve-cache/models/convaiinnovations/laya/{Revision}/another.safetensors?etag=test"
+ })
+ {
+ using var directory = new TemporaryDirectory();
+ using var http = new HttpClient(new FakeHandler((request, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.TemporaryRedirect)
+ {
+ RequestMessage = request,
+ Headers = { Location = new Uri(cachePath, UriKind.Relative) }
+ })));
+
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), http, CancellationToken.None));
+ Assert.False(File.Exists(Path.Combine(directory.Path, Revision, "manifest.json")));
+ }
+ }
+
+ [Fact]
+ public async Task DownloadAsync_RejectsDefaultRevisionHashMismatchBeforePublishingAsset()
+ {
+ using var directory = new TemporaryDirectory();
+ using var http = new HttpClient(new FakeHandler((request, _) => Task.FromResult(Success(request, Encoding.UTF8.GetBytes("wrong upstream bytes")))));
+
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, ModelManifest.DefaultRevision, ["english"]), http, CancellationToken.None));
+
+ Assert.False(File.Exists(Path.Combine(directory.Path, ModelManifest.DefaultRevision, "manifest.json")));
+ Assert.False(File.Exists(Path.Combine(directory.Path, ModelManifest.DefaultRevision, "hub", "model.safetensors")));
+ Assert.Empty(Directory.EnumerateFiles(Path.Combine(directory.Path, ModelManifest.DefaultRevision), "*.tmp", SearchOption.AllDirectories));
+ }
+
+ [Fact]
+ public async Task DownloadAsync_UsesSeparateRevisionDirectories()
+ {
+ using var directory = new TemporaryDirectory();
+ using var http = new HttpClient(new FakeHandler((request, _) => Task.FromResult(Success(request))));
+ var secondRevision = new string('b', 40);
+
+ var first = await HuggingFaceDownloader.DownloadAsync(new DownloadOptions(directory.Path, Revision, ["english"]), http, CancellationToken.None);
+ var second = await HuggingFaceDownloader.DownloadAsync(new DownloadOptions(directory.Path, secondRevision, ["english"]), http, CancellationToken.None);
+
+ Assert.NotEqual(first, second);
+ Assert.True(File.Exists(first));
+ Assert.True(File.Exists(second));
+ }
+
+ [Fact]
+ public async Task DownloadAsync_FailedUpdateLeavesPreviousManifestUntouched()
+ {
+ using var directory = new TemporaryDirectory();
+ using var successfulHttp = new HttpClient(new FakeHandler((request, _) => Task.FromResult(Success(request))));
+ var manifestPath = await HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["english"]), successfulHttp, CancellationToken.None);
+ var originalManifest = await File.ReadAllBytesAsync(manifestPath);
+ using var failingHttp = new HttpClient(new FakeHandler((request, _) => Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
+ {
+ RequestMessage = request
+ })));
+
+ await Assert.ThrowsAsync(() => HuggingFaceDownloader.DownloadAsync(
+ new DownloadOptions(directory.Path, Revision, ["multilingual"]), failingHttp, CancellationToken.None));
+
+ Assert.Equal(originalManifest, await File.ReadAllBytesAsync(manifestPath));
+ Assert.Empty(Directory.EnumerateFiles(Path.Combine(directory.Path, Revision), "manifest.json.*.tmp"));
+ }
+
+ private static HttpResponseMessage Success(HttpRequestMessage request, byte[]? content = null)
+ {
+ var file = GetModelFile(request.RequestUri!);
+ return SuccessForFile(request, file, content);
+ }
+
+ private static HttpResponseMessage SuccessForFile(HttpRequestMessage request, string file, byte[]? content = null)
+ {
+ var bytes = content ?? FixtureContent(file);
+ return new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ RequestMessage = request,
+ Content = new ByteArrayContent(bytes)
+ };
+ }
+
+ private static string GetModelFile(Uri uri)
+ => string.Join('/', GetSourceFile(uri).Split('/').SkipWhile(segment => segment is "multilingual" or "typed-decisions"));
+
+ private static string GetSourceFile(Uri uri)
+ {
+ var path = uri.AbsolutePath;
+ const string resolveMarker = "/resolve/";
+ const string cacheMarker = "/api/resolve-cache/models/convaiinnovations/laya/";
+ var marker = path.Contains(resolveMarker, StringComparison.Ordinal) ? resolveMarker :
+ path.Contains(cacheMarker, StringComparison.Ordinal) ? cacheMarker : null;
+ if (marker is null) return path.TrimStart('/');
+ var remainder = path[(path.IndexOf(marker, StringComparison.Ordinal) + marker.Length)..];
+ var revisionSeparator = remainder.IndexOf('/', StringComparison.Ordinal);
+ return Uri.UnescapeDataString(revisionSeparator < 0 ? remainder : remainder[(revisionSeparator + 1)..]);
+ }
+
+ private static byte[] FixtureContent(string file)
+ => file == "tokenizer/tokenizer_config.json"
+ ? Encoding.UTF8.GetBytes("{\"tokenizer_class\":\"TokenizersBackend\",\"backend\":\"legacy\",\"is_local\":true,\"extra_special_tokens\":[\"a\",\"b\"]}")
+ : Encoding.UTF8.GetBytes($"fixture:{file}");
+
+ private sealed class FakeHandler(Func> responder) : HttpMessageHandler
+ {
+ public List Requests { get; } = [];
+
+ protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+ {
+ Requests.Add(request.RequestUri!);
+ return responder(request, cancellationToken);
+ }
+ }
+
+ private sealed class TemporaryDirectory : IDisposable
+ {
+ public string Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+
+ public TemporaryDirectory() => Directory.CreateDirectory(Path);
+
+ public void Dispose() => Directory.Delete(Path, recursive: true);
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/LayaService.Tests.csproj b/tools/laya_service/tests/LayaService.Tests.csproj
new file mode 100644
index 00000000..4ed37102
--- /dev/null
+++ b/tools/laya_service/tests/LayaService.Tests.csproj
@@ -0,0 +1,19 @@
+
+
+ net10.0
+ false
+ true
+ OpenClaw.LayaService.Tests
+ $(NoWarn);xUnit1051
+
+
+
+
+
+
+ all
+ runtime; build; native; contentfiles; analyzers; buildtransitive
+
+
+
+
diff --git a/tools/laya_service/tests/ModelManifestTests.cs b/tools/laya_service/tests/ModelManifestTests.cs
new file mode 100644
index 00000000..c842ac30
--- /dev/null
+++ b/tools/laya_service/tests/ModelManifestTests.cs
@@ -0,0 +1,117 @@
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using OpenClaw.LayaService.Models;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class ModelManifestTests
+{
+ private const string Revision = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+ [Fact]
+ public void LoadAndVerify_AcceptsEveryCheckpointAndReturnsAbsoluteAssets()
+ {
+ using var directory = new TemporaryDirectory();
+ var manifestPath = WriteManifest(directory.Path, Revision, ["english", "multilingual", "typed-decisions"]);
+
+ var verified = ModelManifest.LoadAndVerify(manifestPath, Revision);
+
+ Assert.Equal(Revision, verified.Revision);
+ Assert.Equal(new[] { "english", "multilingual", "typed-decisions" }, verified.Checkpoints.Keys.Order().ToArray());
+ foreach (var checkpoint in verified.Checkpoints.Values)
+ {
+ Assert.True(Path.IsPathFullyQualified(checkpoint.AbsolutePath));
+ Assert.Equal(ModelManifest.ModelFiles.Count, checkpoint.FileHashes.Count);
+ }
+ }
+
+ [Fact]
+ public void LoadAndVerify_RejectsWrongVersionRevisionAndUnknownCheckpoint()
+ {
+ using var directory = new TemporaryDirectory();
+ var path = WriteManifest(directory.Path, Revision, ["english"]);
+ var original = File.ReadAllText(path);
+
+ File.WriteAllText(path, original.Replace("\"version\":1", "\"version\":2", StringComparison.Ordinal));
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, Revision));
+
+ File.WriteAllText(path, original);
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, new string('b', 40)));
+
+ var unknown = JsonDocument.Parse(original).RootElement;
+ var badManifest = new
+ {
+ version = 1,
+ revision = Revision,
+ checkpoints = new Dictionary
+ {
+ ["unknown"] = unknown.GetProperty("checkpoints").GetProperty("english").Clone()
+ }
+ };
+ File.WriteAllText(path, JsonSerializer.Serialize(badManifest));
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, Revision));
+ }
+
+ [Fact]
+ public void LoadAndVerify_RejectsPathTraversalMissingAndModifiedAssets()
+ {
+ using var directory = new TemporaryDirectory();
+ var path = WriteManifest(directory.Path, Revision, ["english"]);
+ var original = File.ReadAllText(path);
+ var manifest = JsonDocument.Parse(original).RootElement;
+ var checkpoint = manifest.GetProperty("checkpoints").GetProperty("english");
+ var hashes = JsonSerializer.Deserialize>(checkpoint.GetProperty("sha256"))!;
+
+ File.WriteAllText(path, JsonSerializer.Serialize(new
+ {
+ version = 1,
+ revision = Revision,
+ checkpoints = new { english = new { path = "../outside", sha256 = hashes } }
+ }));
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, Revision));
+
+ File.WriteAllText(path, original);
+ File.Delete(Path.Combine(directory.Path, "english", ModelManifest.ModelFiles[0]));
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, Revision));
+
+ WriteManifest(directory.Path, Revision, ["english"]);
+ File.AppendAllText(Path.Combine(directory.Path, "english", ModelManifest.ModelFiles[0]), "changed");
+ Assert.Throws(() => ModelManifest.LoadAndVerify(path, Revision));
+ }
+
+ internal static string WriteManifest(string root, string revision, IReadOnlyList checkpoints)
+ {
+ var checkpointData = new Dictionary(StringComparer.Ordinal);
+ foreach (var checkpoint in checkpoints)
+ {
+ var checkpointPath = Path.Combine(root, checkpoint);
+ Directory.CreateDirectory(checkpointPath);
+ var hashes = new Dictionary(StringComparer.Ordinal);
+ foreach (var file in ModelManifest.ModelFiles)
+ {
+ var filePath = Path.Combine(checkpointPath, file.Replace('/', Path.DirectorySeparatorChar));
+ Directory.CreateDirectory(Path.GetDirectoryName(filePath)!);
+ var contents = Encoding.UTF8.GetBytes($"fixture:{checkpoint}:{file}");
+ File.WriteAllBytes(filePath, contents);
+ hashes.Add(file, Convert.ToHexStringLower(SHA256.HashData(contents)));
+ }
+
+ checkpointData.Add(checkpoint, new { path = checkpoint, sha256 = hashes });
+ }
+
+ var manifestPath = Path.Combine(root, "manifest.json");
+ File.WriteAllText(manifestPath, JsonSerializer.Serialize(new { version = 1, revision, checkpoints = checkpointData }));
+ return manifestPath;
+ }
+
+ private sealed class TemporaryDirectory : IDisposable
+ {
+ public string Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+
+ public TemporaryDirectory() => Directory.CreateDirectory(Path);
+
+ public void Dispose() => Directory.Delete(Path, recursive: true);
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/NLayaDecisionPredictorTests.cs b/tools/laya_service/tests/NLayaDecisionPredictorTests.cs
new file mode 100644
index 00000000..73ef063f
--- /dev/null
+++ b/tools/laya_service/tests/NLayaDecisionPredictorTests.cs
@@ -0,0 +1,185 @@
+using System.Text.Json;
+using OpenClaw.LayaService.Inference;
+using OpenClaw.LayaService.Protocol;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class NLayaDecisionPredictorTests
+{
+ private const string Model = "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982";
+ private const string QuestionsJson = """
+ {"tier":{"type":"choice","instructions":"Choose a tier","criteria":{"small":"simple","large":"complex"}},"severity":{"type":"score","instructions":"Rate severity","criteria":["low","high"]},"urgent":{"type":"noul","instructions":"Is it urgent?","criteria":{"false":"no","true":"yes"}}}
+ """;
+
+ [Theory]
+ [InlineData("A short English message.", null, "english")]
+ [InlineData("你好世界，这是一条包含 English tokens 的混合文本", null, "multilingual")]
+ [InlineData("Bonjour merci. Je suis avec une demande pour un remboursement.", "fr", "multilingual")]
+ [InlineData("中文", "en", "language_checkpoint_conflict")]
+ [InlineData("Hello", "fr", "multilingual")]
+ public void ResolveCheckpoint_UsesLanguageAndRejectsConflictingHints(string state, string? language, string expected)
+ {
+ using var stateDocument = JsonDocument.Parse(JsonSerializer.Serialize(state));
+ using var questionsDocument = JsonDocument.Parse(QuestionsJson);
+ var request = Request(stateDocument.RootElement.Clone(), questionsDocument.RootElement.Clone(), language);
+
+ if (expected == "language_checkpoint_conflict")
+ {
+ var exception = Assert.Throws(() =>
+ NLayaDecisionPredictor.ResolveCheckpoint(request, "auto", Installed("english", "multilingual")));
+ Assert.Equal(expected, exception.ReasonCode);
+ }
+ else
+ {
+ Assert.Equal(expected, NLayaDecisionPredictor.ResolveCheckpoint(request, "auto", Installed("english", "multilingual")));
+ }
+ }
+
+ [Fact]
+ public void ResolveCheckpoint_DoesNotAutoSelectTypedDecisionsAndRequiresExplicitInstallation()
+ {
+ using var stateDocument = JsonDocument.Parse("\"A typed invoice decision\"");
+ using var questionsDocument = JsonDocument.Parse("""
+ {"action":{"type":"choice","instructions":"Choose action","criteria":{"act":"act","review":"review"}},"needs_review":{"type":"noul","instructions":"Needs review?","criteria":{"false":"no","true":"yes"}},"outcome":{"type":"choice","instructions":"Choose outcome","criteria":{"ok":"ok","fail":"fail"}},"risk":{"type":"score","instructions":"Rate risk","criteria":["low","high"]},"urgency":{"type":"score","instructions":"Rate urgency","criteria":["low","high"]}}
+ """);
+ var request = Request(stateDocument.RootElement.Clone(), questionsDocument.RootElement.Clone());
+ var questions = NLaya.Questions.Parse(questionsDocument.RootElement.GetRawText());
+
+ Assert.Equal("agent_trace_observability", NLaya.Routing.Router.MatchTypedDecisionsWorkflow(questions.Keys));
+ Assert.Equal("english", NLayaDecisionPredictor.ResolveCheckpoint(request, "auto", Installed("english", "typed-decisions")));
+ Assert.Equal("typed-decisions", NLayaDecisionPredictor.ResolveCheckpoint(request, "typed-decisions", Installed("typed-decisions")));
+ Assert.Throws(() =>
+ NLayaDecisionPredictor.ResolveCheckpoint(request, "unknown", Installed("english")));
+ using var nonEnglishState = JsonDocument.Parse("\"你好世界\"");
+ var multilingualRequest = Request(nonEnglishState.RootElement.Clone(), questionsDocument.RootElement.Clone());
+ var unavailable = Assert.Throws(() =>
+ NLayaDecisionPredictor.ResolveCheckpoint(multilingualRequest, "auto", Installed("english")));
+ Assert.Equal("checkpoint_not_installed", unavailable.ReasonCode);
+ }
+
+ [Fact]
+ public void DeviceFallbackAndTokenTruncationAreRejected()
+ {
+ NLayaDecisionPredictor.EnsureDevice("torchsharp:cpu", "cpu");
+ Assert.Throws(() => NLayaDecisionPredictor.EnsureDevice("torchsharp:cpu", "cuda"));
+ NLayaDecisionPredictor.EnsureTokenBudget(512, 512);
+ var tooLong = Assert.Throws(() => NLayaDecisionPredictor.EnsureTokenBudget(513, 512));
+ Assert.Equal("input_exceeds_token_budget", tooLong.ReasonCode);
+ Assert.Throws(() => NLayaDecisionPredictor.EnsureReservedTokenAbsent("safe [MASK] input", "[MASK]"));
+ Assert.Throws(() => NLayaDecisionPredictor.EnsureOptionTokenBudget([49, 50]));
+ Assert.Throws(() => NLayaDecisionPredictor.EnsureQuestionHeadBudget(97, [48, 48], 192));
+ }
+
+ [Fact]
+ public void MapPrediction_PreservesTypedAnswersAndAddsPinnedMetadata()
+ {
+ using var stateDocument = JsonDocument.Parse("\"A short English message.\"");
+ using var questionsDocument = JsonDocument.Parse(QuestionsJson);
+ using var predictionDocument = JsonDocument.Parse("""
+ {"answers":{"tier":{"type":"choice","choice":"small","confidence":0.5,"answer_confidence":0.7,"probabilities":{"small":0.7,"large":0.3}},"severity":{"type":"score","score":0.3,"probabilities":{"0":0.7,"1":0.3}},"urgent":{"type":"noul","noul":0.8,"value":true}},"usage":{"input_tokens":12,"output_tokens":0}}
+ """);
+ var request = Request(stateDocument.RootElement.Clone(), questionsDocument.RootElement.Clone(), rubricVersion: "rubric-v1");
+
+ var root = NLayaDecisionPredictor.MapPrediction(
+ request, predictionDocument.RootElement, "english", "cpu", "uncalibrated");
+
+ Assert.Equal(Model, root.GetProperty("model").GetString());
+ Assert.Equal(root.GetProperty("answers").GetRawText(), root.GetProperty("raw_answers").GetRawText());
+ Assert.Equal("small", root.GetProperty("answers").GetProperty("tier").GetProperty("choice").GetString());
+ Assert.Equal(0.3, root.GetProperty("answers").GetProperty("severity").GetProperty("probabilities").GetProperty("1").GetDouble());
+ Assert.True(root.GetProperty("answers").GetProperty("urgent").GetProperty("value").GetBoolean());
+ var metadata = root.GetProperty("metadata");
+ Assert.Equal("1.0.0", metadata.GetProperty("sdk_version").GetString());
+ Assert.Equal("NLaya", metadata.GetProperty("runtime").GetString());
+ Assert.Equal(Model[5..], metadata.GetProperty("revision").GetString());
+ Assert.Equal("english", metadata.GetProperty("checkpoint").GetString());
+ Assert.Equal("rubric-v1", metadata.GetProperty("rubric_version").GetString());
+ Assert.Equal(StrictJson.SchemaHash(request.Questions), metadata.GetProperty("schema_hash").GetString());
+ Assert.False(metadata.GetProperty("truncated").GetBoolean());
+ }
+
+ [Fact]
+ public void CalibrationV2TransformsAnswersAndRetainsRawProbabilities()
+ {
+ using var stateDocument = JsonDocument.Parse("\"A short English message.\"");
+ using var questionsDocument = JsonDocument.Parse(QuestionsJson);
+ using var predictionDocument = JsonDocument.Parse("""
+ {"answers":{"tier":{"type":"choice","choice":"small","confidence":0.5,"answer_confidence":0.7,"probabilities":{"small":0.7,"large":0.3}},"severity":{"type":"score","score":0.3,"probabilities":{"0":0.7,"1":0.3}},"urgent":{"type":"noul","noul":0.8,"value":true}},"usage":{"input_tokens":12,"output_tokens":0}}
+ """);
+ var request = Request(stateDocument.RootElement.Clone(), questionsDocument.RootElement.Clone());
+ var artifact = JsonSerializer.SerializeToUtf8Bytes(new
+ {
+ version = 2,
+ model = Model,
+ schema_hash = StrictJson.SchemaHash(request.Questions),
+ sdk_version = "1.0.0",
+ runtime = "NLaya",
+ temperatures = new Dictionary>
+ {
+ ["english"] = new() { ["choice:2"] = 2, ["score:2"] = 2, ["noul:2"] = 2 }
+ },
+ validation = new { }
+ });
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".json");
+ File.WriteAllBytes(path, artifact);
+ try
+ {
+ var calibration = CalibrationStore.Load(path, Model);
+ var calibrated = calibration.Apply(request, "english", predictionDocument.RootElement.GetProperty("answers"));
+ var response = NLayaDecisionPredictor.MapPrediction(
+ request, predictionDocument.RootElement, "english", "cpu", calibration.Identifier, calibrated);
+
+ Assert.Equal(predictionDocument.RootElement.GetProperty("answers").GetRawText(), response.GetProperty("raw_answers").GetRawText());
+ Assert.NotEqual(0.7, response.GetProperty("answers").GetProperty("tier").GetProperty("probabilities").GetProperty("small").GetDouble());
+ Assert.NotEqual(0.8, response.GetProperty("answers").GetProperty("urgent").GetProperty("noul").GetDouble());
+ }
+ finally
+ {
+ File.Delete(path);
+ }
+ }
+
+ [Fact]
+ public void CalibrationV2AcceptsRoundedProbabilitySums()
+ {
+ using var stateDocument = JsonDocument.Parse("\"A short English message.\"");
+ using var questionsDocument = JsonDocument.Parse(QuestionsJson);
+ using var answersDocument = JsonDocument.Parse("""
+ {"tier":{"type":"choice","choice":"small","probabilities":{"small":0.701,"large":0.3}},"severity":{"type":"score","score":0.3,"probabilities":{"0":0.7,"1":0.3}},"urgent":{"type":"noul","noul":0.8,"value":true}}
+ """);
+ var request = Request(stateDocument.RootElement.Clone(), questionsDocument.RootElement.Clone());
+ var artifact = JsonSerializer.SerializeToUtf8Bytes(new
+ {
+ version = 2,
+ model = Model,
+ schema_hash = StrictJson.SchemaHash(request.Questions),
+ sdk_version = "1.0.0",
+ runtime = "NLaya",
+ temperatures = new Dictionary>
+ {
+ ["english"] = new() { ["choice:2"] = 2, ["score:2"] = 2, ["noul:2"] = 2 }
+ },
+ validation = new { }
+ });
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".json");
+ File.WriteAllBytes(path, artifact);
+ try
+ {
+ var calibration = CalibrationStore.Load(path, Model);
+ var calibrated = calibration.Apply(request, "english", answersDocument.RootElement);
+ var expected = Math.Sqrt(0.701) / (Math.Sqrt(0.701) + Math.Sqrt(0.3));
+
+ Assert.Equal(expected, calibrated.GetProperty("tier").GetProperty("probabilities").GetProperty("small").GetDouble(), 12);
+ }
+ finally
+ {
+ File.Delete(path);
+ }
+ }
+
+ private static DecisionWireRequest Request(JsonElement state, JsonElement questions, string? language = null, string rubricVersion = "rubric-v1")
+ => new() { Model = Model, State = state, Questions = questions, RubricVersion = rubricVersion, Language = language };
+
+ private static IReadOnlySet Installed(params string[] names) => new HashSet(names, StringComparer.Ordinal);
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/ProtocolTests.cs b/tools/laya_service/tests/ProtocolTests.cs
new file mode 100644
index 00000000..96788c5b
--- /dev/null
+++ b/tools/laya_service/tests/ProtocolTests.cs
@@ -0,0 +1,112 @@
+using System.Text;
+using System.Text.Json;
+using OpenClaw.LayaService.Protocol;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class ProtocolTests
+{
+ private const string Model = "laya@1c5edc17a7acd8701df6fc341c0d179f1c62c982";
+ private const string GoldenRequest = "{\"model\":\"" + Model + "\",\"state\":\"hello\",\"questions\":{\"tier\":{\"type\":\"choice\",\"instructions\":\"Which task?\",\"criteria\":{\"small\":\"simple\",\"large\":\"complex\"}}},\"rubric_version\":\"test-v1\"}";
+
+ [Fact]
+ public void ParseAndValidate_AcceptsGoldenRequestAndPreservesQuestionOrder()
+ {
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest));
+
+ RequestValidator.Validate(request, Model);
+ Assert.Equal("hello", request.State.GetString());
+ Assert.Equal("small", request.Questions.GetProperty("tier").GetProperty("criteria").EnumerateObject().First().Name);
+ Assert.Equal("{\"tier\":{\"type\":\"choice\",\"instructions\":\"Which task?\",\"criteria\":{\"small\":\"simple\",\"large\":\"complex\"}}}",
+ StrictJson.Canonicalize(request.Questions));
+ }
+
+ [Fact]
+ public void ParseRequest_RejectsDuplicateKeysNonFiniteNumbersAndInvalidUtf8()
+ {
+ foreach (var json in new[]
+ {
+ "{\"model\":\"a\",\"model\":\"b\"}",
+ "{\"x\":NaN}",
+ "{\"questions\":{\"tier\":{\"criteria\":{},\"criteria\":{}}}}"
+ })
+ {
+ var error = Assert.Throws(() => StrictJson.ParseRequest(Encoding.UTF8.GetBytes(json)));
+ Assert.Equal("invalid_json", error.ReasonCode);
+ }
+
+ var invalidUtf8 = new byte[] { 0x7B, 0x22, 0x78, 0x22, 0x3A, 0x22, 0xFF, 0x22, 0x7D };
+ Assert.Throws(() => StrictJson.ParseRequest(invalidUtf8));
+ }
+
+ [Fact]
+ public void SchemaHash_PreservesCandidateOrder()
+ {
+ const string questions = """{"tier":{"type":"choice","instructions":"Which task?","criteria":{"small":"simple","large":"complex"}}}""";
+ const string reversed = """{"tier":{"type":"choice","instructions":"Which task?","criteria":{"large":"complex","small":"simple"}}}""";
+ var first = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest)).Questions;
+ var second = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest.Replace(questions, reversed, StringComparison.Ordinal))).Questions;
+
+ Assert.NotEqual(StrictJson.SchemaHash(first), StrictJson.SchemaHash(second));
+ }
+
+ [Fact]
+ public void Validate_RejectsUnknownFieldsAndMismatchedModelWithSafeReasons()
+ {
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest));
+
+ var modelError = Assert.Throws(() => RequestValidator.Validate(
+ request with { Model = "SECRET" }, Model));
+ Assert.Equal("model_version_mismatch", modelError.ReasonCode);
+
+ var unknown = GoldenRequest[..^1] + ",\"secret\":\"not echoed\"}";
+ var fieldError = Assert.Throws(() => StrictJson.ParseRequest(Encoding.UTF8.GetBytes(unknown)));
+ Assert.Equal("invalid_request", fieldError.ReasonCode);
+ }
+
+ [Fact]
+ public void Validate_EnforcesStateQuestionAndRubricBounds()
+ {
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest));
+ var longState = request with { State = System.Text.Json.JsonDocument.Parse("\"" + new string('x', 32001) + "\"").RootElement.Clone() };
+ Assert.Equal("invalid_state", Assert.Throws(() => RequestValidator.Validate(longState, Model)).ReasonCode);
+
+ var invalidRubric = request with { RubricVersion = "../secret" };
+ Assert.Equal("invalid_rubric", Assert.Throws(() => RequestValidator.Validate(invalidRubric, Model)).ReasonCode);
+ }
+
+ [Fact]
+ public void Validate_RejectsInvalidQuestionTypesAndChoiceBounds()
+ {
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest));
+ var invalidType = JsonDocument.Parse("""{"tier":{"type":"text","instructions":"Choose","criteria":null}}""").RootElement.Clone();
+ var typeError = Assert.Throws(() => RequestValidator.Validate(
+ request with { Questions = invalidType }, Model));
+ Assert.Equal("invalid_question_type", typeError.ReasonCode);
+
+ var tooManyChoices = JsonSerializer.SerializeToElement(new Dictionary
+ {
+ ["tier"] = new
+ {
+ type = "choice",
+ instructions = "Choose",
+ criteria = Enumerable.Range(0, 21).ToDictionary(index => $"choice{index}", _ => "label")
+ }
+ });
+ var choicesError = Assert.Throws(() => RequestValidator.Validate(
+ request with { Questions = tooManyChoices }, Model));
+ Assert.Equal("invalid_choices", choicesError.ReasonCode);
+ }
+
+ [Fact]
+ public void Validate_AcceptsNoulQuestionsWithoutOptionalCriteria()
+ {
+ var request = StrictJson.ParseRequest(Encoding.UTF8.GetBytes(GoldenRequest));
+ var questions = JsonDocument.Parse("""
+ {"high_risk":{"type":"noul","instructions":"Is this high risk?"}}
+ """).RootElement.Clone();
+
+ RequestValidator.Validate(request with { Questions = questions }, Model);
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/ReliabilityPlotTests.cs b/tools/laya_service/tests/ReliabilityPlotTests.cs
new file mode 100644
index 00000000..7e42717a
--- /dev/null
+++ b/tools/laya_service/tests/ReliabilityPlotTests.cs
@@ -0,0 +1,53 @@
+using System.Text.Json;
+using OpenClaw.LayaService.Reporting;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class ReliabilityPlotTests
+{
+ [Fact]
+ public void WritePng_RequiresLabeledProbabilityCohort()
+ {
+ var report = RoutingJournalReport.Summarize([
+ Element("""{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","latency_ms":1}""")
+ ]);
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".png");
+ try
+ {
+ Assert.Throws(() => ReliabilityPlot.WritePng(report, path));
+ Assert.False(File.Exists(path));
+ }
+ finally
+ {
+ if (File.Exists(path)) File.Delete(path);
+ }
+ }
+
+ [Fact]
+ public void WritePng_WritesPngForCalibrationCohort()
+ {
+ var report = RoutingJournalReport.Summarize(
+ [Element("""{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","latency_ms":1,"probabilities":{"T0":0.8,"T1":0.1,"T2":0.05,"T3":0.05}}""")],
+ [Element("""{"decision_id":"a","expected_tier":"T0"}""")]);
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".png");
+ try
+ {
+ ReliabilityPlot.WritePng(report, path);
+ var bytes = File.ReadAllBytes(path);
+
+ Assert.True(bytes.Length > 100);
+ Assert.Equal(new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }, bytes[..8]);
+ }
+ finally
+ {
+ if (File.Exists(path)) File.Delete(path);
+ }
+ }
+
+ private static JsonElement Element(string json)
+ {
+ using var document = JsonDocument.Parse(json);
+ return document.RootElement.Clone();
+ }
+}
\ No newline at end of file
diff --git a/tools/laya_service/tests/RoutingJournalReportTests.cs b/tools/laya_service/tests/RoutingJournalReportTests.cs
new file mode 100644
index 00000000..fb366737
--- /dev/null
+++ b/tools/laya_service/tests/RoutingJournalReportTests.cs
@@ -0,0 +1,109 @@
+using System.Text.Json;
+using OpenClaw.LayaService.Reporting;
+using Xunit;
+
+namespace OpenClaw.LayaService.Tests;
+
+public sealed class RoutingJournalReportTests
+{
+ [Fact]
+ public void Summarize_MeasuresJevFallbackAgainstLabeledDecisions()
+ {
+ var rows = new[]
+ {
+ Element("""{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","proposed_tier":"T0","latency_ms":100,"input_tokens":2000,"estimated_cost_usd":0.000084}"""),
+ Element("""{"decision_id":"b","baseline_tier":"T3","applied_tier":"T3","reason":"timeout","latency_ms":1500}""")
+ };
+ var labels = new[]
+ {
+ Element("""{"decision_id":"a","expected_tier":"T0"}"""),
+ Element("""{"decision_id":"b","expected_tier":"T3","high_risk":true}""")
+ };
+
+ var report = RoutingJournalReport.Summarize(rows, labels).Document;
+ var quality = report["quality"]!;
+
+ Assert.Equal(1, quality["label_coverage"]!.GetValue());
+ Assert.Equal(1, quality["jev_with_fallback"]!["accuracy"]!.GetValue());
+ Assert.Equal(0.5, quality["baseline"]!["accuracy"]!.GetValue());
+ Assert.Equal(0.5, quality["always_t2"]!["under_routing_rate"]!.GetValue());
+ Assert.Equal(1, quality["jev_with_fallback"]!["high_risk_capability_retention"]!.GetValue());
+ Assert.Equal(0.000084, report["estimated_reported_decision_cost_usd"]!.GetValue());
+ Assert.Equal(1500, report["added_latency_ms"]!["p95"]!.GetValue());
+ }
+
+ [Fact]
+ public void Summarize_SplitsLayaCalibrationByCheckpointAndCountsAbstention()
+ {
+ var rows = new[]
+ {
+ LayaRow("0", "english"),
+ LayaRow("1", "multilingual")
+ };
+ var labels = new[] { Element("""{"decision_id":"0","expected_tier":"T0"}"""), Element("""{"decision_id":"1","expected_tier":"T0"}""") };
+
+ var report = RoutingJournalReport.Summarize(rows, labels).Document;
+
+ Assert.Equal(2, report["calibration_quality"]!.AsArray().Count);
+ Assert.Equal(0, report["calibration_quality"]![0]!["accuracy"]!.GetValue());
+ Assert.NotNull(report["quality"]!["decision_with_fallback"]);
+ Assert.Null(report["quality"]!["jev_with_fallback"]);
+ }
+
+ [Fact]
+ public void Summarize_UnlabeledDataDoesNotClaimAccuracy()
+ {
+ var row = Element("""{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","latency_ms":0}""");
+
+ var report = RoutingJournalReport.Summarize([row]).Document;
+
+ Assert.Null(report["quality"]);
+ Assert.Null(report["proposal_disagreement_with_baseline"]);
+ Assert.Empty(report["calibration_quality"]!.AsArray());
+ }
+
+ [Fact]
+ public void Summarize_RejectsEmptyDuplicateAndUnmatchedData()
+ {
+ var row = Element("""{"decision_id":"a","baseline_tier":"T2","applied_tier":"T2","latency_ms":1}""");
+ Assert.Throws(() => RoutingJournalReport.Summarize([]));
+ Assert.Throws(() => RoutingJournalReport.Summarize([row, row]));
+ Assert.Throws(() => RoutingJournalReport.Summarize([row], [Element("""{"decision_id":"unknown","expected_tier":"T0"}""")]));
+ }
+
+ [Fact]
+ public async Task ReadJsonLines_ReportsPathAndLineForMalformedRows()
+ {
+ var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N") + ".jsonl");
+ await File.WriteAllTextAsync(path, "{}\nnot-json\n");
+ try
+ {
+ var exception = Assert.Throws(() => RoutingJournalReport.ReadJsonLines(path));
+ Assert.Contains(path + ":2: invalid JSON", exception.Message, StringComparison.Ordinal);
+ }
+ finally
+ {
+ File.Delete(path);
+ }
+ }
+
+ private static JsonElement LayaRow(string id, string checkpoint)
+ => Element(JsonSerializer.Serialize(new
+ {
+ decision_id = id,
+ baseline_tier = "T2",
+ applied_tier = "T2",
+ latency_ms = 1,
+ provider = "laya",
+ model = "laya@revision",
+ rubric_version = "v1",
+ metadata = new { checkpoint, calibration_id = "raw" },
+ probabilities = new Dictionary { ["T0"] = 0.1, ["T1"] = 0.1, ["T2"] = 0.1, ["T3"] = 0.1, ["abstain"] = 0.6 }
+ }));
+
+ private static JsonElement Element(string json)
+ {
+ using var document = JsonDocument.Parse(json);
+ return document.RootElement.Clone();
+ }
+}
\ No newline at end of file