feat(evals): build and publish trace-derived evals locally

schemas/README.md

@@ -3,6 +3,38 @@
 Versioned JSON Schemas for artifacts that cross surface boundaries (desktop
 app, skills, CLI, ladder). One spine, adopted everywhere.
 
+## Local workload eval authoring
+
+The `understudy.eval-project.v2`, export-proof, execution-index-row, metric,
+coverage, harness, environment, splits, check-fixtures, check-report, and approval schemas define the private
+coding-agent workspace checked by `understudy evals check`. The workload
+profile remains Markdown; its exact bytes are bound by both intent approval and
+the deterministic check-input hash. These contracts require a provider-free
+local environment replay, independent good/wrong evidence, explicit lineage
+coverage, and a separate post-check owner approval.
+
+The `understudy.eval-publication.v1` and `understudy.eval-release.v1` JSON
+Schemas define the structural hosted boundary for this workflow. Publication
+carries the checked hashes, a compact backend-verifiable source attestation,
+final approval, executable layout, and bundle inventory. The server response
+adds the immutable release seal. Neither contract contains raw source traces,
+the expiring export receipt, the local export-proof file, or mutable authoring
+state.
+
+Within the private project manifest, `source.export_proof_sha256` binds the
+exact local export-proof file. Within the check report and hosted
+publication/release source, that field instead binds the exact opaque
+`source_attestation` token. The CLI verifies both links before upload, and the
+backend verifies the attestation itself before sealing a release.
+
+These Draft 2020-12 schemas do not express the release contract's cross-field
+path rules. Consumers must also parse publications with the package's exported
+`EvalPublicationSchema` and releases with `EvalReleaseSchema`. Those semantic
+validators require a unique, code-unit-sorted bundle inventory, every declared
+artifact and entrypoint, disjoint executable roots, entrypoints inside their
+declared roots, and no undeclared files outside the executable module trees.
+The CLI applies them before upload and to every hosted response.
+
 ## Outcome-first replacement contracts
 
 Four draft-2020-12 contracts form the fail-closed evidence boundary for an

schemas/understudy.eval-approval.v1.schema.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://understudylabs.com/schemas/understudy.eval-approval.v1.schema.json",
+  "title": "understudy.eval-approval.v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "approver", "intent_confirmed_at", "workload_profile_sha256", "metric_sha256"],
+  "properties": {
+    "schema_version": { "const": "understudy.eval-approval.v1" },
+    "approver": { "type": "string", "minLength": 1 },
+    "intent_confirmed_at": { "$ref": "#/$defs/timestamp" },
+    "workload_profile_sha256": { "$ref": "#/$defs/sha" },
+    "metric_sha256": { "$ref": "#/$defs/sha" },
+    "approved_at": { "$ref": "#/$defs/timestamp" },
+    "eval_set_sha256": { "$ref": "#/$defs/sha" },
+    "coverage_sha256": { "$ref": "#/$defs/sha" },
+    "environment_sha256": { "$ref": "#/$defs/sha" },
+    "verifier_sha256": { "$ref": "#/$defs/sha" },
+    "check_report_sha256": { "$ref": "#/$defs/sha" }
+  },
+  "oneOf": [
+    { "required": ["approved_at", "eval_set_sha256", "coverage_sha256", "environment_sha256", "verifier_sha256", "check_report_sha256"] },
+    {
+      "not": {
+        "anyOf": [
+          { "required": ["approved_at"] },
+          { "required": ["eval_set_sha256"] },
+          { "required": ["coverage_sha256"] },
+          { "required": ["environment_sha256"] },
+          { "required": ["verifier_sha256"] },
+          { "required": ["check_report_sha256"] }
+        ]
+      }
+    }
+  ],
+  "$defs": {
+    "timestamp": { "type": "string", "format": "date-time", "pattern": "Z$" },
+    "sha": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
+  }
+}

schemas/understudy.eval-check-fixtures.v1.schema.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://understudylabs.com/schemas/understudy.eval-check-fixtures.v1.schema.json",
+  "title": "understudy.eval-check-fixtures.v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "representative", "known_good", "intentionally_wrong"],
+  "properties": {
+    "schema_version": { "const": "understudy.eval-check-fixtures.v1" },
+    "representative": { "$ref": "#/$defs/good" },
+    "known_good": { "$ref": "#/$defs/good" },
+    "intentionally_wrong": { "$ref": "#/$defs/wrong" }
+  },
+  "$defs": {
+    "nonempty": { "type": "string", "minLength": 1 },
+    "path": { "type": "string", "minLength": 1, "pattern": "^(?!/)(?![A-Za-z]:[\\\\/])(?!.*(?:^|/)\\.\\.(?:/|$))[^\\\\]+$" },
+    "evidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "reference", "statement"],
+      "properties": {
+        "kind": { "enum": ["owner_confirmation", "terminal_state_receipt", "workload_invariant"] },
+        "reference": { "$ref": "#/$defs/nonempty" },
+        "statement": { "$ref": "#/$defs/nonempty" }
+      }
+    },
+    "good": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "candidate", "correctness_evidence"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "candidate": { "$ref": "#/$defs/path" },
+        "state": { "$ref": "#/$defs/path" },
+        "correctness_evidence": { "$ref": "#/$defs/evidence" }
+      }
+    },
+    "wrong": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "candidate", "incorrectness_evidence"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "candidate": { "$ref": "#/$defs/path" },
+        "state": { "$ref": "#/$defs/path" },
+        "incorrectness_evidence": { "$ref": "#/$defs/evidence" }
+      }
+    }
+  }
+}

schemas/understudy.eval-check.v1.schema.json

@@ -0,0 +1,112 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://understudylabs.com/schemas/understudy.eval-check.v1.schema.json",
+  "title": "understudy.eval-check.v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "checked_at", "status", "task_count", "representative_replay", "oracle_fixture", "wrong_fixture", "source", "check_input_sha256", "eval_set_sha256", "coverage_sha256", "environment_sha256", "verifier_sha256"],
+  "properties": {
+    "schema_version": { "const": "understudy.eval-check.v1" },
+    "checked_at": { "$ref": "#/$defs/timestamp" },
+    "status": { "const": "passed" },
+    "task_count": { "type": "integer", "minimum": 1 },
+    "representative_replay": { "$ref": "#/$defs/representative" },
+    "oracle_fixture": { "$ref": "#/$defs/passed" },
+    "wrong_fixture": { "$ref": "#/$defs/rejected" },
+    "source": { "$ref": "#/$defs/source" },
+    "check_input_sha256": { "$ref": "#/$defs/sha" },
+    "eval_set_sha256": { "$ref": "#/$defs/sha" },
+    "coverage_sha256": { "$ref": "#/$defs/sha" },
+    "environment_sha256": { "$ref": "#/$defs/sha" },
+    "verifier_sha256": { "$ref": "#/$defs/sha" }
+  },
+  "$defs": {
+    "nonempty": { "type": "string", "minLength": 1 },
+    "timestamp": { "type": "string", "format": "date-time", "pattern": "Z$" },
+    "sha": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scope", "scope_sha256", "index_sha256", "export_proof_sha256", "capture_count", "size_bytes"],
+      "properties": {
+        "scope": { "$ref": "#/$defs/scope" },
+        "scope_sha256": { "$ref": "#/$defs/sha" },
+        "index_sha256": { "$ref": "#/$defs/sha" },
+        "export_proof_sha256": { "$ref": "#/$defs/sha" },
+        "capture_count": { "type": "integer", "minimum": 0 },
+        "size_bytes": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "scope": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["schema_version", "selector", "org_id", "project_id", "workload_id", "from", "to", "ingestion_cutoff"],
+      "properties": {
+        "schema_version": { "const": "understudy.export-scope.v1" },
+        "selector": { "const": "workload-window" },
+        "org_id": { "$ref": "#/$defs/nonempty" },
+        "project_id": { "$ref": "#/$defs/nonempty" },
+        "workload_id": { "$ref": "#/$defs/nonempty" },
+        "from": { "$ref": "#/$defs/timestamp" },
+        "to": { "$ref": "#/$defs/timestamp" },
+        "ingestion_cutoff": { "$ref": "#/$defs/timestamp" }
+      }
+    },
+    "evidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "reference", "statement"],
+      "properties": {
+        "kind": { "enum": ["owner_confirmation", "terminal_state_receipt", "workload_invariant"] },
+        "reference": { "$ref": "#/$defs/nonempty" },
+        "statement": { "$ref": "#/$defs/nonempty" }
+      }
+    },
+    "representative": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "evidence", "candidate_sha256", "state_sha256", "replay_sha256", "result", "feedback", "provider_called"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "evidence": { "$ref": "#/$defs/evidence" },
+        "candidate_sha256": { "$ref": "#/$defs/sha" },
+        "state_sha256": { "oneOf": [{ "$ref": "#/$defs/sha" }, { "type": "null" }] },
+        "replay_sha256": { "$ref": "#/$defs/sha" },
+        "result": { "const": "passed" },
+        "feedback": { "$ref": "#/$defs/nonempty" },
+        "provider_called": { "const": false }
+      }
+    },
+    "passed": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "evidence", "candidate_sha256", "state_sha256", "replay_sha256", "result", "feedback"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "evidence": { "$ref": "#/$defs/evidence" },
+        "candidate_sha256": { "$ref": "#/$defs/sha" },
+        "state_sha256": { "oneOf": [{ "$ref": "#/$defs/sha" }, { "type": "null" }] },
+        "replay_sha256": { "$ref": "#/$defs/sha" },
+        "result": { "const": "passed" },
+        "feedback": { "$ref": "#/$defs/nonempty" }
+      }
+    },
+    "rejected": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "evidence", "candidate_sha256", "state_sha256", "replay_sha256", "result", "feedback"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "evidence": { "$ref": "#/$defs/evidence" },
+        "candidate_sha256": { "$ref": "#/$defs/sha" },
+        "state_sha256": { "oneOf": [{ "$ref": "#/$defs/sha" }, { "type": "null" }] },
+        "replay_sha256": { "$ref": "#/$defs/sha" },
+        "result": { "const": "rejected" },
+        "feedback": { "$ref": "#/$defs/nonempty" }
+      }
+    }
+  }
+}

schemas/understudy.eval-coverage.v1.schema.json

@@ -0,0 +1,57 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://understudylabs.com/schemas/understudy.eval-coverage.v1.schema.json",
+  "title": "understudy.eval-coverage.v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "lineage", "execution_modes", "failure_classes"],
+  "properties": {
+    "schema_version": { "const": "understudy.eval-coverage.v1" },
+    "lineage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["execution_index_sha256", "counts"],
+      "properties": {
+        "execution_index_sha256": { "$ref": "#/$defs/sha" },
+        "counts": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["complete", "ambiguous", "unlinked"],
+          "properties": {
+            "complete": { "$ref": "#/$defs/count" },
+            "ambiguous": { "$ref": "#/$defs/count" },
+            "unlinked": { "$ref": "#/$defs/count" }
+          }
+        }
+      }
+    },
+    "execution_modes": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/entry" } },
+    "failure_classes": { "type": "array", "items": { "$ref": "#/$defs/entry" } }
+  },
+  "$defs": {
+    "sha": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "count": { "type": "integer", "minimum": 0 },
+    "entry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "observed_count", "task_ids", "disposition"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "observed_count": { "$ref": "#/$defs/count" },
+        "task_ids": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } },
+        "disposition": { "enum": ["covered", "owner_accepted_uncovered"] },
+        "owner_note": { "type": "string", "minLength": 1 }
+      },
+      "allOf": [
+        {
+          "if": { "properties": { "disposition": { "const": "covered" } }, "required": ["disposition"] },
+          "then": { "properties": { "task_ids": { "minItems": 1 } } }
+        },
+        {
+          "if": { "properties": { "disposition": { "const": "owner_accepted_uncovered" } }, "required": ["disposition"] },
+          "then": { "required": ["owner_note"], "properties": { "task_ids": { "maxItems": 0 } } }
+        }
+      ]
+    }
+  }
+}

schemas/understudy.eval-draft-check-fixtures.v1.schema.json

@@ -0,0 +1,52 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://understudylabs.com/schemas/understudy.eval-draft-check-fixtures.v1.schema.json",
+  "title": "understudy.eval-draft-check-fixtures.v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "representative", "known_good", "intentionally_wrong"],
+  "properties": {
+    "schema_version": { "const": "understudy.eval-draft-check-fixtures.v1" },
+    "representative": { "$ref": "#/$defs/good" },
+    "known_good": { "$ref": "#/$defs/good" },
+    "intentionally_wrong": { "$ref": "#/$defs/wrong" }
+  },
+  "$defs": {
+    "nonempty": { "type": "string", "minLength": 1 },
+    "path": { "type": "string", "minLength": 1, "pattern": "^(?!/)(?![A-Za-z]:[\\\\/])(?!.*(?:^|/)\\.\\.(?:/|$))[^\\\\]+$" },
+    "evidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "reference", "statement"],
+      "properties": {
+        "kind": { "enum": ["owner_confirmation", "terminal_state_receipt", "workload_invariant", "agent_inference"] },
+        "reference": { "$ref": "#/$defs/nonempty" },
+        "statement": { "$ref": "#/$defs/nonempty" }
+      }
+    },
+    "good": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "candidate", "correctness_evidence"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "candidate": { "$ref": "#/$defs/path" },
+        "state": { "$ref": "#/$defs/path" },
+        "correctness_evidence": { "$ref": "#/$defs/evidence" }
+      }
+    },
+    "wrong": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["task_id", "input_provenance", "candidate", "incorrectness_evidence"],
+      "properties": {
+        "task_id": { "$ref": "#/$defs/nonempty" },
+        "input_provenance": { "$ref": "#/$defs/nonempty" },
+        "candidate": { "$ref": "#/$defs/path" },
+        "state": { "$ref": "#/$defs/path" },
+        "incorrectness_evidence": { "$ref": "#/$defs/evidence" }
+      }
+    }
+  }
+}