From 32236b71621fe26fa214935d8eed88b0af23d966 Mon Sep 17 00:00:00 2001 From: Tabata Jahoda <294592495+thabatarj8-stack@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:14:46 -0300 Subject: [PATCH 1/2] Validate experimental standard artifacts in CI --- .github/workflows/quality.yml | 3 + README.md | 5 +- README.pt-BR.md | 5 +- scripts/validate_experiment_002.py | 96 ++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 scripts/validate_experiment_002.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 7776442..51fc41e 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -16,11 +16,14 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.13" + - name: Install schema validator + run: python -m pip install jsonschema==4.25.1 - name: Validate public artifacts run: | python -m json.tool codemeta.json >/dev/null python -m json.tool reproducibility/schema/trajectory.schema.json >/dev/null python scripts/check_local_links.py + python scripts/validate_experiment_002.py python reproducibility/scripts/validate.py - name: Run the synthetic integration fixture run: | diff --git a/README.md b/README.md index 6d5be09..366adcc 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ When an agent uses tools, changes state, or acts in an external system, a plausi |---|---| | Understand the idea in five minutes | [Why final answers are not enough](articles/en/why-final-answers-are-not-enough.md) | | Inspect the evaluation design | [Methodology](docs/METHODOLOGY.md) | +| Inspect the versioned operational contracts | [Experimental Standard v0.1](docs/experimental-standard/v0.1/README.md) | | Run the public calibration fixture | [Reproducibility package](reproducibility/README.md) | | Check what the evidence does and does not support | [Evidence status](docs/EVIDENCE-STATUS.md) | | Review the experiment history | [Experiment registry](experiments/README.md) | @@ -71,8 +72,8 @@ Nirmata is a pilot research program and case laboratory. It is not yet a benchma ```text articles/ Accessible essays in English and Portuguese case-studies/ Applied operational cases -docs/ Method, evidence ledger, literature map, and agenda -experiments/ Experiment registry and status +docs/ Method, evidence ledger, literature map, agenda, and experimental standard +experiments/ Experiment registry, protocols, schemas, and status reproducibility/ Schema, synthetic traces, evaluator, scoring, and tests ``` diff --git a/README.pt-BR.md b/README.pt-BR.md index 7a5a30c..104621b 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -14,6 +14,7 @@ Quando um agente usa ferramentas, altera estados ou atua em um sistema externo, |---|---| | Entender a ideia em cinco minutos | [Por que a resposta final não basta](articles/pt-BR/por-que-a-resposta-final-nao-basta.md) | | Inspecionar o desenho de avaliação | [Metodologia](docs/METHODOLOGY.pt-BR.md) | +| Inspecionar os contratos operacionais versionados | [Padrão Experimental v0.1](docs/experimental-standard/v0.1/README.md) | | Executar a calibração pública | [Pacote de reprodutibilidade](reproducibility/README.pt-BR.md) | | Ver o que as evidências sustentam ou não | [Estado das evidências](docs/EVIDENCE-STATUS.pt-BR.md) | | Revisar o histórico experimental | [Registro de experimentos](experiments/README.md) | @@ -71,8 +72,8 @@ O Nirmata é um programa piloto de pesquisa e laboratório de casos. Ainda não ```text articles/ Ensaios acessíveis em português e inglês case-studies/ Casos operacionais aplicados -docs/ Método, evidências, literatura e agenda -experiments/ Registro e estado dos experimentos +docs/ Método, evidências, literatura, agenda e padrão experimental +experiments/ Registro, protocolos, schemas e estado dos experimentos reproducibility/ Schema, trajetórias sintéticas, avaliador, pontuação e testes ``` diff --git a/scripts/validate_experiment_002.py b/scripts/validate_experiment_002.py new file mode 100644 index 0000000..9e832d2 --- /dev/null +++ b/scripts/validate_experiment_002.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Validate Experiment 002 schemas and the public template artifacts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from jsonschema import Draft202012Validator, FormatChecker + + +ROOT = Path(__file__).resolve().parents[1] +EXPERIMENT = ROOT / "experiments" / "002" +SCHEMAS = EXPERIMENT / "schemas" + + +def load_json(path: Path) -> object: + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def validator(name: str) -> Draft202012Validator: + schema = load_json(SCHEMAS / name) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema, format_checker=FormatChecker()) + + +def validate_jsonl(path: Path, artifact_validator: Draft202012Validator) -> int: + count = 0 + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + artifact = json.loads(line) + errors = sorted(artifact_validator.iter_errors(artifact), key=lambda error: list(error.path)) + if errors: + locations = ", ".join( + f"line {line_number} /{'/'.join(map(str, error.path)) or ''}: {error.message}" + for error in errors + ) + raise ValueError(f"{path.relative_to(ROOT)}: {locations}") + count += 1 + if count == 0: + raise ValueError(f"{path.relative_to(ROOT)} contains no predictions") + return count + + +def main() -> None: + trace_validator = validator("trace.schema.json") + prediction_validator = validator("prediction.schema.json") + manifest_validator = validator("run-manifest.schema.json") + + minimal_trace = { + "trace_id": "NIR-002-SCHEMA-CHECK", + "experiment_id": "002", + "system_id": "schema-check", + "run_id": "NIR-002-SCHEMA-CHECK", + "final_output": {"type": "text", "content": "synthetic validation fixture"}, + "events": [ + { + "event_id": "event-001", + "sequence": 0, + "actor": "agent", + "event_type": "model_output", + } + ], + } + trace_validator.validate(minimal_trace) + + manifest = load_json(EXPERIMENT / "manifests" / "run-manifest.template.json") + manifest_validator.validate(manifest) + + content_count = validate_jsonl( + EXPERIMENT / "predictions" / "content-predictions.template.jsonl", + prediction_validator, + ) + trajectory_count = validate_jsonl( + EXPERIMENT / "predictions" / "trajectory-predictions.template.jsonl", + prediction_validator, + ) + expected_count = manifest["dataset"]["trace_count"] + if content_count != expected_count or trajectory_count != expected_count: + raise ValueError( + "prediction template counts must match dataset.trace_count " + f"({expected_count}); received content={content_count}, trajectory={trajectory_count}" + ) + + load_json(EXPERIMENT / "manifests" / "freeze-record.template.json") + print( + "Experiment 002 artifacts: ok " + f"({content_count} content predictions, {trajectory_count} trajectory predictions)" + ) + + +if __name__ == "__main__": + main() From 5bc9a18762552fad78668150209250f9af9b23a7 Mon Sep 17 00:00:00 2001 From: Tabata Jahoda <294592495+thabatarj8-stack@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:15:46 -0300 Subject: [PATCH 2/2] Fix experimental standard Markdown endings --- docs/experimental-standard/v0.1/BLINDING-PROTOCOL.md | 2 +- docs/experimental-standard/v0.1/PUBLICATION-SAFETY.md | 2 +- docs/experimental-standard/v0.1/README.md | 2 +- docs/experimental-standard/v0.1/RUN-MANIFEST.md | 2 +- docs/experimental-standard/v0.1/SCORING-SPEC.md | 2 +- docs/experimental-standard/v0.1/TRACE-SCHEMA.md | 2 +- experiments/002/VALIDATION-NOTES.md | 2 +- experiments/002/evaluators/content-evaluator-v1.md | 2 +- experiments/002/evaluators/trajectory-evaluator-v1.md | 2 +- experiments/002/holdout/README.md | 2 +- experiments/002/predictions/README.md | 2 +- experiments/002/protocol/protocol-v1.0.md | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/experimental-standard/v0.1/BLINDING-PROTOCOL.md b/docs/experimental-standard/v0.1/BLINDING-PROTOCOL.md index ba0fa91..7478bde 100644 --- a/docs/experimental-standard/v0.1/BLINDING-PROTOCOL.md +++ b/docs/experimental-standard/v0.1/BLINDING-PROTOCOL.md @@ -60,4 +60,4 @@ Confirmatory predictions are not manually repaired after ground-truth reveal. Er ## Evidence downgrade -Evidence classification may be downgraded after execution but never upgraded beyond the class permitted by the frozen protocol. \ No newline at end of file +Evidence classification may be downgraded after execution but never upgraded beyond the class permitted by the frozen protocol. diff --git a/docs/experimental-standard/v0.1/PUBLICATION-SAFETY.md b/docs/experimental-standard/v0.1/PUBLICATION-SAFETY.md index 146dbca..b5239c5 100644 --- a/docs/experimental-standard/v0.1/PUBLICATION-SAFETY.md +++ b/docs/experimental-standard/v0.1/PUBLICATION-SAFETY.md @@ -48,4 +48,4 @@ Secrets and credentials must never be committed to the public repository, includ ## Publication review -Before any evidence artifact becomes public, verify that it contains no unnecessary identity data, credentials, private communications, or operational details that increase risk without increasing scientific auditability. \ No newline at end of file +Before any evidence artifact becomes public, verify that it contains no unnecessary identity data, credentials, private communications, or operational details that increase risk without increasing scientific auditability. diff --git a/docs/experimental-standard/v0.1/README.md b/docs/experimental-standard/v0.1/README.md index 75d23ff..6684cff 100644 --- a/docs/experimental-standard/v0.1/README.md +++ b/docs/experimental-standard/v0.1/README.md @@ -21,4 +21,4 @@ This directory defines the first public experimental standard for Nirmata. It op 5. Evidence classifications may be downgraded after execution, but not upgraded beyond what the frozen protocol permits. 6. Public artifacts must disclose only the information necessary to audit the scientific claim. -This standard is intentionally small and provider-independent. It is expected to evolve through versioned releases rather than silent edits. \ No newline at end of file +This standard is intentionally small and provider-independent. It is expected to evolve through versioned releases rather than silent edits. diff --git a/docs/experimental-standard/v0.1/RUN-MANIFEST.md b/docs/experimental-standard/v0.1/RUN-MANIFEST.md index 59eeccb..20e2943 100644 --- a/docs/experimental-standard/v0.1/RUN-MANIFEST.md +++ b/docs/experimental-standard/v0.1/RUN-MANIFEST.md @@ -68,4 +68,4 @@ Record repository commit, runtime information, dependency lockfile/hash, and oth ## Evidence Downgrade Principle -Evidence classification may be downgraded after execution but never upgraded beyond the class permitted by the frozen protocol. A development run cannot become confirmatory merely because its result is favorable. \ No newline at end of file +Evidence classification may be downgraded after execution but never upgraded beyond the class permitted by the frozen protocol. A development run cannot become confirmatory merely because its result is favorable. diff --git a/docs/experimental-standard/v0.1/SCORING-SPEC.md b/docs/experimental-standard/v0.1/SCORING-SPEC.md index 45ab389..88c1772 100644 --- a/docs/experimental-standard/v0.1/SCORING-SPEC.md +++ b/docs/experimental-standard/v0.1/SCORING-SPEC.md @@ -85,4 +85,4 @@ Additional post-hoc analyses are permitted only when clearly labeled exploratory ## Privacy -Scoring artifacts use synthetic trace identifiers. Real identities, credentials, private account identifiers, and unnecessary private communications must not be included. \ No newline at end of file +Scoring artifacts use synthetic trace identifiers. Real identities, credentials, private account identifiers, and unnecessary private communications must not be included. diff --git a/docs/experimental-standard/v0.1/TRACE-SCHEMA.md b/docs/experimental-standard/v0.1/TRACE-SCHEMA.md index beacc84..ab5630d 100644 --- a/docs/experimental-standard/v0.1/TRACE-SCHEMA.md +++ b/docs/experimental-standard/v0.1/TRACE-SCHEMA.md @@ -114,4 +114,4 @@ Initial vocabulary: Crashes, blocked actions, invalid evaluator outputs, aborted runs, and hypothesis-disconfirming results should be preserved rather than silently removed. -A trace is observed evidence, not ground truth. Safety classification belongs to the evaluation and ground-truth layers. \ No newline at end of file +A trace is observed evidence, not ground truth. Safety classification belongs to the evaluation and ground-truth layers. diff --git a/experiments/002/VALIDATION-NOTES.md b/experiments/002/VALIDATION-NOTES.md index ecf3ce1..1a9d356 100644 --- a/experiments/002/VALIDATION-NOTES.md +++ b/experiments/002/VALIDATION-NOTES.md @@ -19,4 +19,4 @@ This note records consistency checks performed before opening the draft pull req This review checks structural consistency of the repository artifacts. It does not constitute confirmatory validation of evaluator behavior, statistical performance, or a real holdout run. -The package remains a pre-freeze draft until the holdout, evaluator configuration, hashes, and freeze record are completed under the protocol. \ No newline at end of file +The package remains a pre-freeze draft until the holdout, evaluator configuration, hashes, and freeze record are completed under the protocol. diff --git a/experiments/002/evaluators/content-evaluator-v1.md b/experiments/002/evaluators/content-evaluator-v1.md index bd1909a..d9e780d 100644 --- a/experiments/002/evaluators/content-evaluator-v1.md +++ b/experiments/002/evaluators/content-evaluator-v1.md @@ -56,4 +56,4 @@ Return exactly one JSON object with no prose before or after it: ## Failure behavior -If the input cannot be evaluated under this contract, return a schema-valid binary judgment according to the frozen experiment policy. Do not invent additional fields. Operational inability to produce a valid response is recorded by the harness as an evaluator failure and is not manually repaired during a confirmatory run. \ No newline at end of file +If the input cannot be evaluated under this contract, return a schema-valid binary judgment according to the frozen experiment policy. Do not invent additional fields. Operational inability to produce a valid response is recorded by the harness as an evaluator failure and is not manually repaired during a confirmatory run. diff --git a/experiments/002/evaluators/trajectory-evaluator-v1.md b/experiments/002/evaluators/trajectory-evaluator-v1.md index 5a702ba..99c521c 100644 --- a/experiments/002/evaluators/trajectory-evaluator-v1.md +++ b/experiments/002/evaluators/trajectory-evaluator-v1.md @@ -75,4 +75,4 @@ Return exactly one JSON object with no prose before or after it: ## Failure behavior -Do not add confidence scores, explanations, alternate labels, or new fields. If the model fails to produce schema-valid output, the harness records an evaluator failure. Confirmatory predictions are not manually repaired. \ No newline at end of file +Do not add confidence scores, explanations, alternate labels, or new fields. If the model fails to produce schema-valid output, the harness records an evaluator failure. Confirmatory predictions are not manually repaired. diff --git a/experiments/002/holdout/README.md b/experiments/002/holdout/README.md index 7b62962..fb06719 100644 --- a/experiments/002/holdout/README.md +++ b/experiments/002/holdout/README.md @@ -90,4 +90,4 @@ Before the holdout is marked `FROZEN`: ## Burned holdouts -Any holdout whose answer key becomes available to an evaluator before prediction freeze is burned for blind confirmatory use with that evaluator context. It may still be retained for development, debugging, demonstration, or regression testing. \ No newline at end of file +Any holdout whose answer key becomes available to an evaluator before prediction freeze is burned for blind confirmatory use with that evaluator context. It may still be retained for development, debugging, demonstration, or regression testing. diff --git a/experiments/002/predictions/README.md b/experiments/002/predictions/README.md index 4079733..c03355d 100644 --- a/experiments/002/predictions/README.md +++ b/experiments/002/predictions/README.md @@ -10,4 +10,4 @@ For a real run: 4. record those hashes in the run manifest and freeze record; 5. do not overwrite or manually repair confirmatory predictions after reveal. -Templates use synthetic identifiers only and contain no ground truth. \ No newline at end of file +Templates use synthetic identifiers only and contain no ground truth. diff --git a/experiments/002/protocol/protocol-v1.0.md b/experiments/002/protocol/protocol-v1.0.md index 95f9c80..5b00950 100644 --- a/experiments/002/protocol/protocol-v1.0.md +++ b/experiments/002/protocol/protocol-v1.0.md @@ -63,4 +63,4 @@ A valid pilot may support a bounded claim about incremental trajectory signal in ## Evidence classification -The intended evidence class is `CONFIRMATORY`. Final classification is determined after execution and may be downgraded, including to `INVALIDATED`, if protocol requirements are violated. \ No newline at end of file +The intended evidence class is `CONFIRMATORY`. Final classification is determined after execution and may be downgraded, including to `INVALIDATED`, if protocol requirements are violated.