From 321f10bc27007e8cb62d5d76e09234ec7d232d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 13:01:54 +0300 Subject: [PATCH 01/15] docs: add SPSS frontend expansion roadmap --- ROADMAP.md | 16 ++ docs/spss-frontend-roadmap.md | 142 +++++++++++++ ...26-08-21-spss-frontend-expansion-design.md | 188 ++++++++++++++++++ 3 files changed, 346 insertions(+) create mode 100644 docs/spss-frontend-roadmap.md create mode 100644 docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md diff --git a/ROADMAP.md b/ROADMAP.md index b862739..8d98fb2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -65,6 +65,22 @@ Prerequisite: the SPSS profile, fixtures, and reference-adapter lessons are matu this does not block specification publication. - [ ] Evaluate additional language frontends as separate adapters that lower to the same canonical plan. +- [ ] Expand the bounded SPSS frontend through the independently versioned + milestones in the [SPSS frontend roadmap](docs/spss-frontend-roadmap.md), + without creating a general SPSS compatibility claim. +- [ ] Publish Frontend 0.3 as syntax-only expansion over immutable Plan 0.1/0.2: + comments, dictionary-order variable lists, grouped supported forms, ordinary + not-equal/NOT predicates, open recode ranges, and additive value labels. +- [ ] Define typed numeric expressions, deterministic numeric/date functions, + missing-value predicates and metadata, additional numeric/date formats, and + restricted conditional blocks in a new Plan/Frontend/In-Place Binding + generation. +- [ ] Define string and physical-schema operations separately, with exact + add/rename/drop-column atomicity and rollback evidence for every claimed SQL + profile. +- [ ] Keep case-count/order and group/cross-dataset operations in separate + profiles whose invariants do not weaken the current same-cases in-place + binding. ## 8. Microsoft SQL Server dialect diff --git a/docs/spss-frontend-roadmap.md b/docs/spss-frontend-roadmap.md new file mode 100644 index 0000000..bbdcd17 --- /dev/null +++ b/docs/spss-frontend-roadmap.md @@ -0,0 +1,142 @@ +# SPSS Syntax Frontend Roadmap + +## Status and boundary + +This roadmap describes intended work, not a compatibility claim or release-date +promise. The current published boundary remains the exact bounded grammar in +[SPSS Syntax Frontend 0.2](spss-syntax-frontend-profile-0.2.md). Every syntax +form outside a claimed frontend contract must continue to fail closed. + +OpenStatSpec will expand the frontend in independently versioned stages. It will +not advertise general IBM SPSS Statistics compatibility, forward source +expressions into SQL, or weaken the same-dataset/same-table and caller-owned +Dolt history rules. + +## Command classification + +| Capability | Intended boundary | Reason | +| --- | --- | --- | +| Comments, existing-variable `TO`, grouped supported forms | Frontend 0.3, Plan 0.1/0.2 | Compile-time syntax expansion only | +| `NOT`, `NE`, `<>`, `~=` over the existing predicate grammar | Frontend 0.3, Plan 0.2 | Canonically lowers to existing comparisons and booleans | +| `LOWEST`/`HIGHEST` recode ranges | Frontend 0.3, Plan 0.1/0.2 | Lowers to finite binary64 endpoints; system missing remains distinct | +| `ADD VALUE LABELS` | Frontend 0.3, Plan 0.1/0.2 | Lowers ordered metadata state to complete label replacement | +| Arithmetic, deterministic numeric/date functions, missing predicates | New Plan/Frontend/Binding generation | Adds expression shapes and normative evaluation semantics | +| `MISSING VALUES` and additional numeric/date formats | New Plan/Frontend/Binding generation | Adds catalog operations and format semantics | +| Restricted `DO IF` | New Plan/Frontend/Binding generation | Requires branch-once and ordered-body semantics | +| `NUMERIC`, `STRING`, string expressions/functions, rename/delete | Separate Plan/Frontend/Binding generation | Adds types, physical DDL, catalog cleanup, and atomicity rules | +| `SELECT IF`, `SORT CASES` | Separate case-transformation profile | Changes current case-count/order invariants | +| Restricted add-variable aggregate and keyed lookup | Separate group/cross-dataset profile | Introduces group and multi-dataset semantics | +| `TEMPORARY`, `FILTER`, procedures, state/order-dependent functions | Explicit non-goal | Requires procedure/session state outside the mutation frontend | + +## Milestone 0: integrity and classification + +- Add specification-owned validation for Plan, Frontend, and In-Place schemas, + manifests, references, canonical bytes, and source/plan hashes. +- Keep release status consistent across profile documents, roadmap, changelog, + tags, and releases. +- Classify every proposed command as frontend-only, new Plan/Binding work, a + separate profile, or an explicit non-goal before accepting syntax. + +## Milestone 1: Frontend 0.3 over Plan 0.1/0.2 + +Frontend 0.3 should add syntax depth without adding plan operations: + +- `COMMENT`, leading-star comments, and inline block comments; +- existing-variable lists with dictionary-order `TO` expansion; +- grouped forms of supported labels, formats, levels, and recodes; +- `NOT`, `NE`, `<>`, and `~=` over the existing predicate grammar with + three-valued truth semantics; +- `LOWEST` and `HIGHEST` recode ranges; and +- `ADD VALUE LABELS`, lowered to a complete value-label replacement using the + ordered metadata state initialized from the request and updated by preceding + commands. + +The manifest must list only Plan 0.1 and Plan 0.2 outputs and prove that all +inherited programs retain their exact prior plans and hashes. No SQL binding or +executor change belongs in this milestone. + +## Milestone 2: typed numeric expressions and metadata + +Define a new Plan, Frontend, and In-Place Binding generation for: + +- arithmetic parentheses, unary minus, `+`, `-`, `*`, and `/`; +- a deterministic numeric-function whitelist; +- `MISSING`, `SYSMIS`, `NMISS`, and `NVALID`; +- `MISSING VALUES` metadata; +- numeric `E`, `COMMA`, `DOT`, and `PCT` formats; and +- deterministic date construction, extraction, arithmetic, and display over + SPSS numeric serial values; and +- restricted `DO IF`/`ELSE IF`/`ELSE` blocks. + +The canonical expression model must define precedence, binary64 behavior, +domain errors, missing propagation, and sequential evaluation independently of +backend SQL behavior. Random, current-time, order-dependent, and +locale-dependent functions remain excluded. A conditional-block branch is +selected once from row state at block entry; its body then executes in source +order. + +## Milestone 3: strings and physical schema + +Define a separate contract milestone for: + +- `NUMERIC` and `STRING` declarations; +- string assignment, comparison, and a small deterministic function whitelist; +- simultaneous `RENAME VARIABLES`; and +- controlled `DELETE VARIABLES` with dependency checks and complete catalog + cleanup. + +The contract must specify fixed-width strings, blanks, padding, truncation, +Unicode indexing, collation, initialization, and physical/catalog identity. +Every SQL profile needs exact add/rename/drop-column atomicity and rollback +evidence. Non-atomic profiles retain explicit pre-provisioning for new targets. +Rename and delete fail before mutation where native DDL cannot satisfy the +contract; copied or staging tables are not recovery mechanisms. Dolt branch, +HEAD, clean-working-set, and caller-owned commit policy remain in force. + +## Milestone 4: case operations + +Treat row-count and row-order changes as a separate case-transformation profile: + +- `SELECT IF` may be considered as an explicit destructive operation with + affected-case evidence and exact UNKNOWN handling. +- `SORT CASES` waits for a persistent logical case-order model with stable + tie-breaking, missing ordering, and string collation. + +Neither command may be retrofitted into a binding that claims to preserve case +count and order. Physical SQL row order is not logical case order. + +## Milestone 5: group and cross-dataset operations + +Only bounded, row-count-preserving forms are candidates: + +- `AGGREGATE OUTFILE=* MODE=ADDVARIABLES` with a small deterministic function + set; and +- a unique-key lookup that adds selected columns to the active dataset without + changing its row count. + +Full `AGGREGATE OUTFILE`, unrestricted `MATCH FILES`, persistent outputs, +staging tables, snapshots, rollback copies, and hidden recovery datasets remain +outside this roadmap. + +## Explicit non-goals + +- Full SPSS syntax compatibility or statistical procedures. +- `TEMPORARY`, `FILTER`, weighting, `SPLIT FILE`, or procedure-scoped state. +- Random, current-time, `LAG`, cumulative, or environment-dependent functions. +- Macros, loops, vectors, extension-language execution, unrestricted joins, + reshaping, or `ALTER TYPE`. +- Any OpenStatSpec-managed persistent version or rollback layer. Dolt remains + the sole persistent version/history mechanism for successful edits. + +## Evidence sources + +The priorities are based on IBM's +[Command Syntax Reference](https://www.ibm.com/docs/en/SSLVMB_25.0.0/pdf/en/IBM_SPSS_Statistics_Command_Syntax_Reference.pdf), +[string-function documentation](https://www.ibm.com/docs/en/spss-statistics/25.0.0?topic=expressions-string-functions), +[FORMATS rules](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=formats-syntax-rules-command), +[SELECT IF reference](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-select-if), +[Aggregate Data documentation](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=transformations-aggregate-data), +and [MATCH FILES reference](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-match-files). +GNU PSPP and UCLA OARC documentation provide secondary implementation and +usage evidence; IBM semantics and OpenStatSpec's normative contracts remain +authoritative. diff --git a/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md new file mode 100644 index 0000000..a2e8d14 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md @@ -0,0 +1,188 @@ +# SPSS Frontend Expansion Design + +## Goal + +Expand the bounded SPSS syntax frontend in independently claimable stages +without promising general IBM SPSS Statistics compatibility, changing published +Plan 0.1/0.2 identities, or weakening the in-place and Dolt ownership +contracts. + +## Research basis + +The design compares the current Frontend 0.2 grammar and conformance artifacts +with IBM's command reference and focused data-management references from GNU +PSPP and UCLA OARC. The highest-value gaps are comments, variable-list syntax, +ordinary predicates, arithmetic, missing-value handling, deterministic +functions, string preparation, and selected dictionary operations. + +The current plan vocabulary creates a hard version boundary. Comments, +variable-list expansion, alternate predicate syntax, open-ended recode ranges, +and additive value-label syntax can lower to unchanged Plan 0.1/0.2 operations. +Arithmetic, functions, missing predicates, string expressions, schema changes, +case deletion, ordering, grouping, and joins require new normative semantics. + +Primary references: + +- [IBM SPSS Statistics Command Syntax Reference](https://www.ibm.com/docs/en/SSLVMB_25.0.0/pdf/en/IBM_SPSS_Statistics_Command_Syntax_Reference.pdf) +- [IBM COMMENT command](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=comment-overview-command) +- [IBM string functions](https://www.ibm.com/docs/en/spss-statistics/25.0.0?topic=expressions-string-functions) +- [IBM SELECT IF](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-select-if) +- [IBM FORMATS syntax rules](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=formats-syntax-rules-command) +- [IBM date and time arithmetic](https://www.ibm.com/docs/SSLVMB_sub/statistics_reference_project_ddita/spss/base/syn_date_and_time_arithmetic_operations_date_time_variables.html) +- [IBM Aggregate Data](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=transformations-aggregate-data) +- [IBM MATCH FILES](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-match-files) + +Secondary references are the [GNU PSPP manual](https://www.gnu.org/software/pspp/manual/) +and [UCLA OARC SPSS resources](https://stats.oarc.ucla.edu/spss/). + +## Selected approach + +Use a contract staircase instead of one broad release. + +1. Frontend 0.3 accepts only syntax that deterministically emits an exact Plan + 0.1 or Plan 0.2 object. +2. A later Plan, Frontend, and In-Place Binding release adds typed numeric + expressions and dictionary metadata together. +3. String and physical-schema operations form a separate contract milestone + with independent SQL atomicity evidence. +4. Case-count/order operations and group/cross-dataset operations use separate + profiles rather than stretching the current same-cases in-place contract. + +This approach creates useful progress before SQL executor changes, keeps old +canonical bytes and hashes immutable, and prevents experimental adapter +identifiers from becoming de facto specification contracts. + +## Contract architecture + +Every accepted syntax expansion increments the frontend contract because the +contract selects accepted grammar, diagnostics, source spans, and output-plan +selection. A frontend-only release may still emit older plan contracts. + +A new Plan contract is required when an addition introduces a new operation, +expression shape, type rule, evaluation rule, or meaning for an existing +operation. A new In-Place Binding contract is required when SQL execution, +catalog mutation, audit behavior, rollback, or profile capability rules change. + +Older schemas, fixtures, canonical JSON, hashes, and diagnostics remain +immutable. New manifests inherit successful and failing older cases and prove +that old source continues to emit the same old plans. + +## Milestones + +### 0. Specification integrity and classification + +- Align release-status wording across roadmap, profile documents, changelog, + tags, and releases. +- Extend repository validation to cover Plan, Frontend, and In-Place manifests, + schemas, unique IDs, references, canonical bytes, and hashes. +- Publish a command-classification table assigning each proposed SPSS form to + Frontend-only, new Plan/Binding, a separate profile, or explicit non-goal. + +### 1. Frontend 0.3: syntax depth over Plan 0.1/0.2 + +- Command comments (`COMMENT` and leading `*`) and inline block comments. +- Existing-variable lists and dictionary-order `TO` expansion. +- Grouped forms of already supported labels, formats, levels, and recodes. +- `NOT`, `NE`, `<>`, and `~=` over the existing comparison/boolean grammar, + with explicit three-valued semantics and canonical lowering to existing + predicate nodes. +- `LOWEST`/`HIGHEST` recode ranges with finite canonical endpoints. +- `ADD VALUE LABELS`, compiled to a complete ordered + `replace_value_labels` operation using ordered metadata state initialized + from the request and updated by preceding commands. + +Frontend 0.3 lists only Plan 0.1 and Plan 0.2 as outputs. It does not add an +executor operation or change an in-place binding. + +### 2. Typed numeric expressions and dictionary metadata + +- Parentheses, unary minus, `+`, `-`, `*`, and `/` with defined precedence, + overflow, division, and system-missing behavior. +- A deterministic numeric-function whitelist, initially `ABS`, `MOD`, `TRUNC`, + `RND`, `SQRT`, `LN`, `LG10`, `EXP`, `MIN`, `MAX`, `SUM`, and `MEAN`. +- `MISSING`, `SYSMIS`, `NMISS`, and `NVALID`, distinguishing system missing + from metadata-defined user missing. +- `MISSING VALUES` as a catalog-only operation. +- Numeric output formats `E`, `COMMA`, `DOT`, and `PCT`; locale-dependent + currency formats remain excluded. +- Deterministic date construction, extraction, arithmetic, and display based on + SPSS numeric serial values; current-time and locale-ambiguous parsing remain + excluded. +- Restricted `DO IF`/`ELSE IF`/`ELSE` containing only operations in the same + contract. A branch condition is evaluated once against the row state at block + entry, while selected body operations retain their source order. + +The Plan expression IR is typed and source-neutral. Adapters must not insert +source expressions into backend SQL or inherit backend coercion, function, +locale, collation, or error behavior. + +### 3. String expressions and schema operations + +- `NUMERIC` and `STRING` declarations with exact initialization and width + semantics. +- String assignment and comparison with explicit fixed-width, blank, Unicode, + padding, truncation, and collation rules. +- A deterministic string-function whitelist: `CONCAT`, `UPCASE`, `LOWER`, + `LTRIM`, `RTRIM`, `LENGTH`, `SUBSTR`, `INDEX`, and `REPLACE`. +- Simultaneous `RENAME VARIABLES` with collision and case-only rename rules. +- Controlled `DELETE VARIABLES` with catalog-reference cleanup and dependency + validation. + +Each SQL profile must publish add, rename, and drop-column atomicity evidence. +MySQL, MariaDB, and Dolt continue to require pre-provisioned new targets where +native add-column DDL cannot satisfy one-transaction apply. Rename and delete +must fail before mutation on a profile whose native DDL cannot satisfy the +contract; an output/staging table or full-table copy is not an allowed +workaround. Dolt commits remain caller-owned. + +### 4. Case operations + +- `SELECT IF` is a destructive, row-count-changing operation with exact TRUE, + FALSE, and UNKNOWN behavior and explicit affected-case evidence. +- `SORT CASES` is deferred until the standard defines persistent logical case + order, stable tie-breaking, missing ordering, and string collation. Physical + table order is never treated as canonical case order. + +These operations use a separate case-transformation profile because the current +in-place binding preserves case count and order. + +### 5. Group and cross-dataset operations + +- Consider only `AGGREGATE OUTFILE=* MODE=ADDVARIABLES` with a small, + deterministic function set and unchanged row count. +- Consider a restricted keyed lookup with one active dataset and one unique-key + lookup dataset, explicit selected columns, and unchanged active row count. + +Full `AGGREGATE OUTFILE`, unrestricted `MATCH FILES`, and any hidden output, +staging, snapshot, rollback, or copied dataset remain forbidden. Dolt is the +sole persistent version/history layer for successful edits. + +## Explicit non-goals + +- No claim of full SPSS syntax compatibility. +- No statistical procedures, procedure output, weighting, `SPLIT FILE`, or + session-wide analysis state. +- No `TEMPORARY` or `FILTER` until a real procedure/query scope exists. +- No random, current-time, `LAG`, cumulative, locale-dependent, or other + order/state-dependent functions in the deterministic expression profile. +- No macros, loops, vectors, extension-language execution, unrestricted joins, + reshaping, or `ALTER TYPE` in these milestones. +- No persistent output/staging copies or OpenStatSpec-managed rollback state. + +## Conformance and release gates + +Each milestone provides a profile document, request/plan schema where +applicable, positive and failure manifest cases, exact source/plan hashes, +stable diagnostics and spans, and inherited compatibility cases. + +Specification validation checks artifact structure and identity. Adapter claims +remain downstream: each adapter pins the exact specification commit and runs +the complete applicable frontend/plan suites. Every binding claim also runs +SQLite plus every claimed PostgreSQL, MySQL, MariaDB, Dolt, or future dialect +service profile, including rollback and forbidden-artifact cases. + +Specification publication precedes adapter claims and does not wait for them. +No adapter may claim an official contract identifier before that contract is +published from an immutable specification commit. Experimental adapter syntax +uses an explicitly adapter-owned identifier until publication and is not +interoperability evidence for the future official contract. From 7a310b531b987ba2d835089795cc8f2dd00534e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 13:40:17 +0300 Subject: [PATCH 02/15] docs: plan SPSS frontend 0.3 specification --- .../plans/2026-08-21-spss-frontend-03.md | 1349 +++++++++++++++++ 1 file changed, 1349 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-21-spss-frontend-03.md diff --git a/docs/superpowers/plans/2026-08-21-spss-frontend-03.md b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md new file mode 100644 index 0000000..2aa1131 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md @@ -0,0 +1,1349 @@ +# SPSS Frontend 0.3 Specification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish a complete specification candidate for SPSS Syntax Frontend 0.3 that adds bounded syntax depth while emitting only immutable Transformation Plan 0.1/0.2 objects. + +**Architecture:** First add repository-owned validation for every Plan, Frontend, and In-Place artifact and repair release-status bookkeeping. Then add an exact Frontend 0.3 request schema, normative profile, inherited-case manifest, and independently authored golden cases. Frontend 0.3 changes accepted source syntax only; it adds no Plan operation, SQL binding, audit schema, or executor behavior. + +**Tech Stack:** Python 3.11+, pytest 8, jsonschema Draft 2020-12, JSON Schema, restricted RFC 8785 canonical JSON, SHA-256, Markdown, GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md` + +## Global Constraints + +- Published Plan 0.1/0.2 schemas, operations, canonical bytes, hashes, and diagnostics are immutable. +- Published Frontend 0.1/0.2 request schemas and manifest cases are immutable. +- Contract identifiers select exact behavior; Frontend 0.3 uses `openstatspec-spss-syntax-frontend-v0.3`. +- Frontend 0.3 emits only `openstatspec-transformation-plan-v0.1` or `openstatspec-transformation-plan-v0.2`. +- A program inside the Plan 0.1 operation subset emits the exact Plan 0.1 object, even when its request contract is Frontend 0.3. +- Comments remain material to `source_hash` but emit no operation. +- Every undefined command, expression, coercion, comment form, or variable-list form fails closed. +- No task adds arithmetic, functions, missing predicates, string expressions, schema operations, case deletion/order, aggregation, joins, persistent copies, or rollback state. +- Specification publication precedes adapter claims; PHP and Python implementation work is planned separately after this contract is reviewed and published. +- Dolt remains caller-owned history; this frontend-only release does not change branch, HEAD, working-set, commit, or pre-provisioning rules. + +## File Structure + +- `src/openstatspec_specification/artifacts.py`: repository artifact loader, JSON Schema validation, canonical hashing, manifest reference validation, inherited-case expansion, and inventory reporting. +- `tests/test_contract_artifacts.py`: positive inventory checks and mutation-based fail-closed tests for schemas, hashes, IDs, references, inheritance, and release status. +- `tools/validate_repository.py`: invokes the artifact validator in addition to existing Dolt and repository controls. +- `pyproject.toml`: declares `jsonschema` and the pytest development dependency. +- `.github/workflows/ci.yml`: installs the package and runs pytest before repository validation. +- `transformation/spss-syntax-frontend-0.3.schema.json`: exact Frontend 0.3 request envelope; structurally identical to 0.2 except contract identity and schema metadata. +- `docs/spss-syntax-frontend-profile-0.3.md`: normative lexical, grammar, lowering, diagnostics, compatibility, and hashing rules. +- `conformance/spss-syntax-frontend-0.3.json`: inherited 0.1/0.2 cases, explicit supersession records, and 35 new independent golden cases. +- `examples/spss-syntax-transformation-0.3.md`: one readable program showing comments, `TO`, grouped format, `NOT`/not-equal, open recode ranges, and additive labels. +- `README.md`, `CHANGELOG.md`, `ROADMAP.md`, `docs/spss-frontend-roadmap.md`: artifact index, release notes, milestone status, and adapter handoff gates. +- `docs/transformation-plan-profile-0.2.md`, `docs/spss-syntax-frontend-profile-0.2.md`, `docs/transformation-plan-sql-binding-0.2.md`: correct published `v0.3.0` status wording only; no normative edits. + +--- + +### Task 1: Add the contract artifact validation foundation + +**Files:** +- Create: `src/openstatspec_specification/artifacts.py` +- Create: `tests/test_contract_artifacts.py` +- Modify: `pyproject.toml` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- Produces: `ArtifactValidationError`, `ArtifactInventory`, `canonical_json_bytes(value: object) -> bytes`, `source_hash(source_text: str) -> str`, and `validate_contract_artifacts(root: Path) -> ArtifactInventory`. +- Consumes: existing files below `transformation/`, `conformance/`, `sql/`, and `docs/`; no adapter code. + +- [ ] **Step 1: Add a failing inventory test.** + +Create `tests/test_contract_artifacts.py` with the existing released inventory as the first executable requirement: + +```python +from __future__ import annotations + +from pathlib import Path + +from openstatspec_specification.artifacts import validate_contract_artifacts + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_released_contract_artifact_inventory_is_valid() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.plan_cases == {"0.1": 4, "0.2": 26} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44} + assert inventory.binding_cases == {"0.1": 6, "0.2": 11} +``` + +- [ ] **Step 2: Run the focused test and confirm the missing-module failure.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: collection fails with `ModuleNotFoundError: No module named 'openstatspec_specification.artifacts'`. + +- [ ] **Step 3: Declare exact validation dependencies and CI setup.** + +Change `pyproject.toml` to include: + +```toml +dependencies = ["jsonschema>=4.23,<5"] + +[project.optional-dependencies] +dev = ["pytest>=8,<9"] +``` + +In `.github/workflows/ci.yml`, insert these commands after `actions/setup-python` and before repository validation: + +```yaml + - run: python -m pip install --disable-pip-version-check -e ".[dev]" + - run: python -m pytest + - run: python tools/validate_repository.py +``` + +Remove the old duplicate standalone `python tools/validate_repository.py` step. + +- [ ] **Step 4: Implement the loader, schema checks, top-level contracts, and unique IDs.** + +Create `src/openstatspec_specification/artifacts.py` with these public types and constants: + +```python +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + + +PLAN_MANIFESTS = { + "0.1": "conformance/transformation-plan-0.1.json", + "0.2": "conformance/transformation-plan-0.2.json", +} +FRONTEND_MANIFESTS = { + "0.1": "conformance/spss-syntax-frontend-0.1.json", + "0.2": "conformance/spss-syntax-frontend-0.2.json", +} +BINDING_MANIFESTS = { + "0.1": "conformance/in-place-transformation-0.1.json", + "0.2": "conformance/in-place-transformation-0.2.json", +} + + +class ArtifactValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class ArtifactInventory: + plan_cases: Mapping[str, int] + frontend_declared_cases: Mapping[str, int] + frontend_effective_cases: Mapping[str, int] + binding_cases: Mapping[str, int] + + +def canonical_json_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def source_hash(source_text: str) -> str: + normalized = source_text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() +``` + +Implement private helpers with these exact responsibilities: + +```python +def _load_json(root: Path, relative: str) -> object: + parts = relative.split("/") + _require( + not relative.startswith("/") + and "\\" not in relative + and all(part not in {"", ".", ".."} for part in parts), + f"noncanonical artifact path: {relative}", + ) + candidate = root.joinpath(*parts) + current = root + for part in parts: + current = current / part + _require(not current.is_symlink(), f"artifact path traverses symlink: {relative}") + _require(candidate.is_file(), f"artifact is missing: {relative}") + try: + return json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactValidationError(message) + +def _resolve_manifest_reference(manifest_relative: str, reference: str) -> str: + _require( + isinstance(reference, str) + and bool(reference) + and not reference.startswith("/") + and "\\" not in reference, + f"noncanonical manifest reference: {reference!r}", + ) + resolved: list[str] = [] + candidate = PurePosixPath(manifest_relative).parent / reference + for part in candidate.parts: + if part in {"", "."}: + continue + if part == "..": + _require(bool(resolved), f"manifest reference escapes repository root: {reference}") + resolved.pop() + continue + resolved.append(part) + _require(bool(resolved), f"manifest reference resolves to repository root: {reference}") + return "/".join(resolved) + +def _require_unique_case_ids(manifest: Mapping[str, object], context: str) -> None: + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{context}: cases must be a list") + seen: set[str] = set() + for case in cases: + _require(isinstance(case, dict), f"{context}: each case must be an object") + case_id = case.get("id") + _require(isinstance(case_id, str) and bool(case_id), f"{context}: case id must be a non-empty string") + _require(case_id not in seen, f"{context}: duplicate case id: {case_id}") + seen.add(case_id) + +def _schema(root: Path, relative: str) -> Mapping[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: schema must be an object") + try: + Draft202012Validator.check_schema(value) + except SchemaError as error: + raise ArtifactValidationError(f"{relative}: invalid JSON Schema: {error.message}") from None + return value +``` + +`validate_contract_artifacts()` must load all six existing manifests, require their exact `manifest_version`, `contract`, and relative schema fields, meta-validate all four referenced Plan/Frontend schemas, require unique case IDs, and return the released counts shown in Step 1. Convert `OSError`, `UnicodeError`, `JSONDecodeError`, `SchemaError`, and `ValidationError` into `ArtifactValidationError` without leaking unrelated filesystem data. + +Resolve every manifest-owned schema or audit reference through +`_resolve_manifest_reference(manifest_relative, reference)` before passing the +result to `_load_json()`. Direct repository-relative loader calls continue to +reject `..`; only the resolver may consume and normalize manifest-relative +parent segments, and it must never permit escape above repository root. + +- [ ] **Step 5: Add fail-closed tests for malformed schemas and duplicate IDs.** + +Add this copy helper and tests: + +```python +import json +import shutil + +import pytest + +from openstatspec_specification.artifacts import ArtifactValidationError + + +def copied_artifacts(tmp_path: Path) -> Path: + for directory in ("conformance", "transformation", "sql", "docs"): + shutil.copytree(ROOT / directory, tmp_path / directory) + shutil.copy2(ROOT / "CHANGELOG.md", tmp_path / "CHANGELOG.md") + shutil.copy2(ROOT / "ROADMAP.md", tmp_path / "ROADMAP.md") + return tmp_path + + +def test_duplicate_manifest_case_id_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"].append(manifest["cases"][0]) + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="duplicate case id"): + validate_contract_artifacts(root) + + +def test_invalid_json_schema_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "transformation/plan-0.1.schema.json" + schema = json.loads(path.read_text(encoding="utf-8")) + schema["type"] = "not-a-json-schema-type" + path.write_text(json.dumps(schema), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="invalid JSON Schema"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 6: Run the focused and full tests.** + +Run: + +```bash +python3 -m pip install --disable-pip-version-check -e ".[dev]" +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +``` + +Expected: all tests pass; inventory remains Plan `4/26`, Frontend `13/44`, Binding `6/11`. + +- [ ] **Step 7: Commit the validation foundation.** + +```bash +git add pyproject.toml .github/workflows/ci.yml src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "test: validate contract artifact inventory" +``` + +### Task 2: Validate canonical hashes and cross-manifest references + +**Files:** +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `tools/validate_repository.py` + +**Interfaces:** +- Consumes: Task 1 `validate_contract_artifacts()`, `canonical_json_bytes()`, and `source_hash()`. +- Produces: exact validation of plan hashes, frontend source hashes, referenced plans, binding references, canonical audit JSON, and failure-case shape. + +- [ ] **Step 1: Add failing mutation tests for plan hash, source hash, and references.** + +```python +@pytest.mark.parametrize( + ("manifest_name", "field"), + [ + ("transformation-plan-0.2.json", "expected_plan_hash"), + ("spss-syntax-frontend-0.2.json", "expected_source_hash"), + ], +) +def test_hash_mismatch_fails_closed( + tmp_path: Path, manifest_name: str, field: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance" / manifest_name + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0][field] = "0" * 64 + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="hash mismatch"): + validate_contract_artifacts(root) + + +def test_missing_frontend_plan_reference_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["expected_plan_case"] = "absent-plan" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="unknown plan case"): + validate_contract_artifacts(root) + + +def test_binding_plan_and_frontend_references_must_agree(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/in-place-transformation-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["applied_plan_case"] = "strict-greater-than" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="binding reference mismatch"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 2: Run the focused tests and confirm all three mutations are accepted incorrectly.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: the three new tests fail because Task 1 does not inspect hashes or references. + +- [ ] **Step 3: Implement Plan manifest validation.** + +For every Plan manifest: + +1. Index cases by ID. +2. Require `expected_error` to be either `null` or a non-empty string. +3. For successful cases, require a `plan` object, validate it against the manifest's schema, calculate `sha256(canonical_json_bytes(plan))`, and require exact lowercase `expected_plan_hash` equality. +4. For failing cases, require `expected_plan_hash` to be absent or `null`; retain the invalid plan as declarative negative input and do not require it to satisfy the plan schema. +5. Require the plan's `contract` to match the manifest contract for successful cases. + +Add private helpers with these signatures: + +```python +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + +def _validate_plan_manifests( + root: Path, + manifests: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, object]]: + """Return version -> case-id -> successful canonical plan case.""" +``` + +- [ ] **Step 4: Implement Frontend request, source hash, and plan-reference validation.** + +For each Frontend case: + +1. Compute `source_hash(request["source_text"])` for every structurally readable request and require exact `expected_source_hash`. +2. Validate successful requests against `request_schema`. +3. For `plan_schema_invalid` failures, require request schema validation to fail; for other failures, require the request schema to pass. +4. Resolve successful expected plans from exactly one of `expected_plan`, `expected_plan_0_1`, `expected_plan_case`, or `expected_plan_case_0_1`. +5. Resolve `expected_plan_case` against the declared Plan manifest and `expected_plan_case_0_1` against Frontend 0.1's resolved expected plans. +6. Validate the resolved plan against the schema selected by its exact `contract`, recompute its canonical hash, and require equality with `expected_plan_hash`. +7. Require failing cases to omit expected plan objects, references, and hashes. + +Use this exact resolver signature: + +```python +def _resolve_frontend_plans( + root: Path, + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, Mapping[str, object]]]: + """Return version -> case-id -> resolved canonical plan for successes.""" +``` + +- [ ] **Step 5: Implement In-Place reference and audit identity validation.** + +For In-Place 0.2 cases containing `applied_plan_case` and +`applied_frontend_case`, require both references to exist, require the frontend +case to resolve to that exact plan case, and verify these audit fields when +present: + +```python +expected_audit["plan_hash"] == plan_case["expected_plan_hash"] +expected_audit["source_hash"] == frontend_case["expected_source_hash"] +expected_audit["canonical_plan_json"].encode("utf-8") == canonical_json_bytes(plan_case["plan"]) +expected_audit["operation_count"] == len(plan_case["plan"]["operations"]) +``` + +Require every In-Place manifest case to have a unique ID and exact manifest +contract. Require every failing binding case to contain a non-empty +`expected_error` and `mutation_started: false`. + +- [ ] **Step 6: Wire artifact validation into the repository command.** + +In `tools/validate_repository.py`, import and call the validator after the +existing reusable Dolt validator and before phrase controls: + +```python +from openstatspec_specification.artifacts import ( # noqa: E402 + ArtifactValidationError, + validate_contract_artifacts, +) + + +if __name__ == "__main__": + try: + main() + inventory = validate_contract_artifacts(ROOT) + validate_repository_controls() + print( + "Validated contract artifacts: " + f"Plan {dict(inventory.plan_cases)}, " + f"Frontend {dict(inventory.frontend_effective_cases)}, " + f"Binding {dict(inventory.binding_cases)}." + ) + except (DoltDeclarationError, ArtifactValidationError) as error: + raise SystemExit(str(error)) from error +``` + +- [ ] **Step 7: Run mutation tests and the complete repository gate.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: all tests pass; the repository command prints all six released inventory counts and exits zero. + +- [ ] **Step 8: Commit canonical artifact validation.** + +```bash +git add src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py tools/validate_repository.py +git commit -m "test: verify transformation artifact identity" +``` + +### Task 3: Repair release status and lock command classification + +**Files:** +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/transformation-plan-profile-0.2.md` +- Modify: `docs/spss-syntax-frontend-profile-0.2.md` +- Modify: `docs/transformation-plan-sql-binding-0.2.md` +- Modify: `CHANGELOG.md` +- Modify: `ROADMAP.md` +- Verify: `docs/spss-frontend-roadmap.md` + +**Interfaces:** +- Consumes: published `v0.3.0` tag target `cd8f198c68b849eb8ed018a894670a0904c2181d` and the command-classification table already in `docs/spss-frontend-roadmap.md`. +- Produces: `validate_release_metadata(root: Path) -> None` and consistent released status for all 0.2 profiles. + +- [ ] **Step 1: Add a failing release-status consistency test.** + +```python +from openstatspec_specification.artifacts import validate_release_metadata + + +def test_v030_profile_status_is_consistently_released() -> None: + validate_release_metadata(ROOT) + + +def test_release_candidate_wording_fails_for_published_profile(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + text = path.read_text(encoding="utf-8").replace( + "Status: released in OpenStatSpec `v0.3.0`", + "Status: release candidate for OpenStatSpec `v0.3.0`", + ) + path.write_text(text, encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="release status"): + validate_release_metadata(root) +``` + +- [ ] **Step 2: Run the test and confirm current release-candidate wording fails.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: `test_v030_profile_status_is_consistently_released` fails because three published 0.2 documents still say release candidate. + +- [ ] **Step 3: Implement exact release metadata validation.** + +Add: + +```python +V030_COMMIT = "cd8f198c68b849eb8ed018a894670a0904c2181d" +V030_PROFILE_DOCUMENTS = ( + "docs/transformation-plan-profile-0.2.md", + "docs/spss-syntax-frontend-profile-0.2.md", + "docs/transformation-plan-sql-binding-0.2.md", +) + + +def validate_release_metadata(root: Path) -> None: + for relative in V030_PROFILE_DOCUMENTS: + text = (root / relative).read_text(encoding="utf-8") + _require( + "Status: released in OpenStatSpec `v0.3.0`" in text, + f"{relative}: release status is inconsistent with v0.3.0", + ) + _require( + "release candidate for the planned OpenStatSpec `v0.3.0`" not in text, + f"{relative}: stale release status remains", + ) + roadmap = (root / "ROADMAP.md").read_text(encoding="utf-8") + _require("`v0.3.0` is the" in roadmap, "ROADMAP release tag mismatch") + _require("current public specification release and is immutable" in roadmap, "ROADMAP release status mismatch") + _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") +``` + +Extend the `tools/validate_repository.py` artifact import with +`validate_release_metadata`, then call `validate_release_metadata(ROOT)` from +`validate_repository_controls()`. + +- [ ] **Step 4: Correct only release bookkeeping text.** + +Replace each 0.2 profile's status paragraph with: + +```markdown +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. +``` + +In `CHANGELOG.md`, replace the conditional release-preparation paragraph under +`v0.3.0` with a factual released statement naming the same tag and commit. Do +not edit any normative command, plan, binding, or diagnostic text. + +- [ ] **Step 5: Record completed PHP downstream evidence without changing specification gates.** + +In `ROADMAP.md` section 9, mark the PHP item complete and cite: + +```markdown +5. [x] PHP pinned specification `v0.3.0` commit + `cd8f198c68b849eb8ed018a894670a0904c2181d`, passed its complete PHP + 8.4/8.5 and PostgreSQL/MySQL/MariaDB/Dolt matrix, and published adapter + [v0.6.0](https://github.com/OpenStatSpec/php/releases/tag/v0.6.0). +``` + +Leave Python status unchanged unless its own exact release evidence is independently verified in that task. + +- [ ] **Step 6: Verify the public command-classification table is complete.** + +Require `docs/spss-frontend-roadmap.md` to contain all four boundary phrases: + +```python +classification = (root / "docs/spss-frontend-roadmap.md").read_text(encoding="utf-8") +for phrase in ( + "Frontend 0.3, Plan 0.1/0.2", + "New Plan/Frontend/Binding generation", + "Separate case-transformation profile", + "Explicit non-goal", +): + _require(phrase in classification, f"SPSS classification is missing {phrase}") +``` + +- [ ] **Step 7: Run all gates and commit bookkeeping.** + +Run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +git diff --check +``` + +Then commit: + +```bash +git add src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py docs/transformation-plan-profile-0.2.md docs/spss-syntax-frontend-profile-0.2.md docs/transformation-plan-sql-binding-0.2.md CHANGELOG.md ROADMAP.md +git commit -m "docs: align v0.3.0 release status" +``` + +### Task 4: Define the Frontend 0.3 request contract and normative grammar + +**Files:** +- Create: `transformation/spss-syntax-frontend-0.3.schema.json` +- Create: `docs/spss-syntax-frontend-profile-0.3.md` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Produces: request contract `openstatspec-spss-syntax-frontend-v0.3` and its exact Draft 2020-12 schema. +- Consumes: unchanged Frontend 0.2 input-schema fields and unchanged Plan 0.1/0.2 contracts. + +- [ ] **Step 1: Add failing schema inventory tests.** + +```python +def test_frontend_03_request_schema_is_registered() -> None: + inventory = validate_contract_artifacts(ROOT) + assert "0.3" in inventory.frontend_declared_cases + + +def test_frontend_03_schema_changes_only_contract_identity() -> None: + old = json.loads((ROOT / "transformation/spss-syntax-frontend-0.2.schema.json").read_text()) + new = json.loads((ROOT / "transformation/spss-syntax-frontend-0.3.schema.json").read_text()) + assert new["$id"].endswith("spss-syntax-frontend-0.3.schema.json") + assert new["title"] == "OpenStatSpec SPSS-like Syntax Frontend Request 0.3" + assert new["properties"]["contract"]["const"] == "openstatspec-spss-syntax-frontend-v0.3" + for document in (old, new): + document.pop("$id") + document.pop("title") + document["properties"]["contract"].pop("const") + assert new == old +``` + +- [ ] **Step 2: Run the test and confirm the 0.3 schema/manifest is absent.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: failure because no Frontend 0.3 schema or registered manifest exists. + +- [ ] **Step 3: Create the exact Frontend 0.3 request schema.** + +Copy Frontend 0.2 structurally and change only: + +```json +{ + "$id": "https://openstatspec.org/schemas/spss-syntax-frontend-0.3.schema.json", + "title": "OpenStatSpec SPSS-like Syntax Frontend Request 0.3", + "properties": { + "contract": {"const": "openstatspec-spss-syntax-frontend-v0.3"} + } +} +``` + +Retain all 0.2 input fields and `$defs`, including ordered variables, storage +kind, variable/value labels, format family/width/decimals, and measurement +level. Do not add missing rules or physical identifiers in 0.3. + +- [ ] **Step 4: Write the normative 0.3 profile with exact grammar boundaries.** + +Create `docs/spss-syntax-frontend-profile-0.3.md` with these sections and rules: + +```text +Status and boundary +Compatibility and output-plan selection +Comments and source identity +Existing-variable lists +Predicate aliases and NOT lowering +Open-ended RECODE ranges +ADD VALUE LABELS +Failure contract +Conformance +``` + +Include this bounded grammar: + +```text +existing-variable-list := existing-variable + ((existing-variable | TO existing-variable))* +comparison-operator := "=" | "<" | "<=" | ">" | ">=" + | "NE" | "<>" | "~=" +predicate := comparison | "(" predicate ")" + | NOT predicate + | predicate AND predicate + | predicate OR predicate +open-range := LOWEST THRU finite-number + | finite-number THRU HIGHEST + | LOWEST THRU HIGHEST +``` + +Normatively require: + +- leading-star comments only when `*` is the first non-whitespace token at a command boundary, ending at the next command period; +- `COMMENT` command comments ending at their command period; +- non-nested `/* ... */` comments wherever whitespace is legal; unterminated or nested forms return `spss_syntax_error`; +- raw comments remain in LF-normalized source hashing and emit no operation; +- a comment-only program returns `spss_syntax_error` because plans cannot have zero operations; +- `TO` expands inclusively by current ordered input-schema state, not by variable spelling or numeric suffix; +- reverse ranges return `invalid_variable_range`; absent endpoints return `unknown_variable`; `TO` is forbidden for generated `INTO` target sequences; +- `NE`, `<>`, and `~=` lower to ordered `< OR >` comparisons; +- `NOT` is accepted only over the existing comparison/boolean grammar and lowers recursively with comparison complements and De Morgan's laws, preserving SQL three-valued truth and canonical same-operator flattening; +- open ranges use `ffefffffffffffff` for negative maximum finite binary64 and `7fefffffffffffff` for positive maximum finite binary64; SQL NULL/system missing is not included; +- `ADD VALUE LABELS` updates an existing typed value at its current ordinal and appends new values in source order, then emits one complete `replace_value_labels` operation per variable; +- comments, `TO`, grouped syntax, open ranges, and additive labels remain in the Plan 0.1 output subset; predicate aliases/`NOT` require existing Plan 0.2 `conditional_assign` syntax. + +- [ ] **Step 5: Register Frontend 0.3 in the validator with an empty draft manifest.** + +Add `"0.3": "conformance/spss-syntax-frontend-0.3.json"` to +`FRONTEND_MANIFESTS`. Create the manifest in Task 5; for this task, keep the +schema/profile changes staged but do not commit until Task 5 supplies a valid +manifest and the repository is green. + +### Task 5: Define inherited compatibility and supersession + +**Files:** +- Create: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Test: `conformance/spss-syntax-frontend-0.1.json` +- Test: `conformance/spss-syntax-frontend-0.2.json` + +**Interfaces:** +- Consumes: Task 4 request schema and profile. +- Produces: `inherited_manifests` semantics and effective-case expansion for Frontend 0.3. + +- [ ] **Step 1: Add failing inheritance tests.** + +```python +def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 2 + assert inventory.frontend_effective_cases["0.3"] == 57 + + +def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["inherited_manifests"][0]["manifest"] = "../outside.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="inherited manifest"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 2: Create the initial Frontend 0.3 manifest.** + +Use this exact top-level shape: + +```json +{ + "manifest_version": "0.3", + "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.3", + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "plan_contracts": [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2" + ], + "request_schema": "../transformation/spss-syntax-frontend-0.3.schema.json", + "plan_schemas": { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json" + }, + "inherited_manifests": [ + { + "manifest": "spss-syntax-frontend-0.1.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" + } + }, + { + "manifest": "spss-syntax-frontend-0.2.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": {} + } + ], + "cases": [ + { + "id": "reject-comment-only-program", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* comment." + }, + "expected_source_hash": "8bfac9af8d0884b68cebc65a42360871592dd6e4e9cbead0bdf7029b6b64f4fc", + "expected_error": "spss_syntax_error" + }, + { + "id": "inline-block-comment-is-ignored", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* comment */." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "3c4ef459e9a2021100ef166ef0038c5f920b126d916db4c7273537d21b256f65", + "expected_error": null + } + ] +} +``` + +The initial effective count is 57: 11 unchanged Frontend 0.1 cases, all 44 +Frontend 0.2 cases, and two declared replacements. Expanded IDs are namespaced +as `0.1/` and `0.2/` so repeated IDs across manifests remain +distinct. + +- [ ] **Step 3: Implement inherited-case expansion and validation.** + +Add: + +```python +def _expand_frontend_cases( + root: Path, + version: str, + manifest: Mapping[str, object], + manifests: Mapping[str, Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + """Return inherited cases with contract override followed by declared cases.""" +``` + +Require inherited paths to be canonical filenames in `conformance/`, target +older registered Frontend manifests only, and use the current manifest's exact +contract as `request_contract_override`. Deep-copy each inherited case and +replace only `request.contract`; source text, expected source hash, plan, +expected plan hash, and expected diagnostic remain unchanged. + +`superseded_cases` maps an inherited case ID to a declared 0.3 replacement case +ID. Require every old ID to exist, every replacement ID to exist in declared +cases, and every replacement to use the same `source_text`. Exclude superseded +old cases from the effective set. Reject inheritance cycles and duplicate +expanded IDs. + +- [ ] **Step 4: Run the focused tests and complete Task 4/5 commit.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend declared counts are `13/44/2`; Frontend effective counts are `13/44/57`. + +Commit: + +```bash +git add transformation/spss-syntax-frontend-0.3.schema.json docs/spss-syntax-frontend-profile-0.3.md conformance/spss-syntax-frontend-0.3.json src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "spec: define SPSS frontend 0.3 contract" +``` + +### Task 6: Add comments, variable lists, and grouped forms + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Consumes: Task 5 inheritance and supersession format. +- Produces: 14 declared cases covering comments and schema-ordered variable-list expansion. + +- [ ] **Step 1: Add a failing exact-count and supersession test.** + +```python +def test_frontend_03_comment_and_varlist_case_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 14 + assert inventory.frontend_effective_cases["0.3"] == 69 +``` + +The effective count is `13 + 44 - 2 superseded + 14 = 69`. + +- [ ] **Step 2: Add the four remaining exact comment cases.** + +Add these case IDs and outcomes: + +```text +comment-command-before-recode success, exact Plan 0.1 recode +leading-star-comment-before-recode success, same Plan 0.1 recode +comments-change-source-hash-not-plan-hash success, same plan with a distinct source hash +reject-unterminated-inline-comment spss_syntax_error +``` + +For each success, embed or reference an exact existing Plan 0.1 case and include +an independently calculated lowercase source hash. A representative case is: + +```json +{ + "id": "leading-star-comment-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "dddbaf0dd69f364ccc895d0d3b65daa63208edfd9b4e8e2112f915f1e73e5d70", + "expected_error": null +} +``` + +Calculate each source hash with: + +```bash +python3 -c 'from openstatspec_specification.artifacts import source_hash; print(source_hash("* recode one to zero.\nRECODE q1 (1 = 0)."))' +``` + +Write the printed lowercase digest into the manifest; do not calculate hashes at adapter runtime. + +- [ ] **Step 3: Verify the two released comment failures remain explicitly superseded.** + +Confirm Frontend 0.1 inheritance metadata remains: + +```json +"superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" +} +``` + +The replacement cases use the exact old source text. The first remains a +failure but changes from unsupported command to empty-program syntax error; the +second becomes a success. Do not edit the released 0.1 manifest. + +- [ ] **Step 4: Add eight variable-list/group cases.** + +Add these exact IDs: + +```text +recode-to-expands-dictionary-order +value-labels-to-expands-dictionary-order +formats-grouped-to-expands-source-order +variable-level-to-expands-source-order +to-sees-variable-created-by-preceding-command +reject-reversed-to-range +reject-unknown-to-endpoint +reject-to-generated-into-targets +``` + +Use input order `a, middle, c, outside` and prove `a TO c` expands to exactly +`a, middle, c`, regardless of spelling. `FORMATS a TO c (F8.2)` emits three +ordered `set_format` operations. `VARIABLE LEVEL a TO c (NOMINAL)` emits three +ordered `set_measurement_level` operations. `COMPUTE appended = a.` followed by +`FORMATS c TO appended (F8.2).` proves current command-order schema state. + +Use stable errors: + +```text +reversed existing-variable range -> invalid_variable_range +absent endpoint -> unknown_variable +TO in generated INTO targets -> spss_syntax_error +``` + +- [ ] **Step 5: Validate all explicit source and plan hashes.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend 0.3 declared/effective counts are `14/69`; no Plan manifest count or hash changes. + +- [ ] **Step 6: Commit comments and variable-list fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py +git commit -m "spec: add frontend comments and variable lists" +``` + +### Task 7: Add predicate aliases and open recode ranges + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` + +**Interfaces:** +- Consumes: unchanged Plan 0.1 range nodes and Plan 0.2 comparison/boolean nodes. +- Produces: 14 declared cases for not-equal/NOT lowering and finite open-range lowering. + +- [ ] **Step 1: Add a failing exact-count test.** + +```python +def test_frontend_03_predicate_and_open_range_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 28 + assert inventory.frontend_effective_cases["0.3"] == 83 +``` + +- [ ] **Step 2: Add eight exact predicate cases.** + +```text +ne-keyword-lowers-to-less-or-greater +angle-not-equal-lowers-to-less-or-greater +tilde-not-equal-lowers-to-less-or-greater +not-equality-lowers-to-less-or-greater +not-nested-and-lowers-by-de-morgan +double-not-retains-canonical-comparison +reject-not-numeric-truthiness +reject-chained-comparison +``` + +Use this exact canonical lowering table in profile and cases: + +```text +NOT (left = right) -> (left < right OR left > right) +NOT (left < right) -> left >= right +NOT (left <= right) -> left > right +NOT (left > right) -> left <= right +NOT (left >= right) -> left < right +NOT (a AND b) -> NOT a OR NOT b +NOT (a OR b) -> NOT a AND NOT b +``` + +`NE`, `<>`, and `~=` all emit the same ordered n-ary `or` node containing `<` +then `>`. Apply existing maximal same-operator flattening after recursive +lowering. Do not constant-fold or reorder. Fail numeric truthiness such as +`IF (NOT source_a) ...` with `expression_type_unsupported`; fail +`a < b < c` with `spss_syntax_error`. + +- [ ] **Step 3: Add six exact open-range cases.** + +```text +recode-lowest-thru-upper +recode-lower-thru-highest +recode-lowest-thru-highest +reject-string-open-range +reject-highest-as-lower-endpoint +reject-lowest-as-upper-endpoint +``` + +Expected finite endpoints are exact typed values: + +```json +{"type": "binary64", "bits": "ffefffffffffffff"} +{"type": "binary64", "bits": "7fefffffffffffff"} +``` + +String open ranges fail `type_mismatch`. `HIGHEST THRU 1` and +`1 THRU LOWEST` fail `spss_syntax_error`. System missing remains unmatched +unless another explicit rule handles `SYSMIS`. + +- [ ] **Step 4: Run exact identity gates.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend 0.3 declared/effective counts are `28/83`; all Plan 0.1/0.2 hashes remain unchanged. + +- [ ] **Step 5: Commit predicate and range fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py docs/spss-syntax-frontend-profile-0.3.md +git commit -m "spec: add frontend predicates and open ranges" +``` + +### Task 8: Add additive value-label semantics + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` + +**Interfaces:** +- Consumes: Frontend request `input_schema.variables[].value_labels` and existing Plan `replace_value_labels`. +- Produces: seven declared cases and deterministic ordered merge semantics. + +- [ ] **Step 1: Add the final failing inventory test.** + +```python +def test_frontend_03_final_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} +``` + +- [ ] **Step 2: Add seven exact additive-label cases.** + +```text +add-value-labels-preserves-existing-and-appends +add-value-labels-replaces-existing-label-in-place +value-labels-then-add-value-labels-uses-ordered-state +add-value-labels-multiple-variable-group +add-string-value-labels +reject-add-value-labels-duplicate-source-value +reject-add-value-labels-type-mismatch +``` + +Use this normative merge algorithm: + +```python +def merge_labels(existing, additions): + result = list(existing) + index = {canonical_typed_value(item["value"]): position for position, item in enumerate(result)} + seen_additions = set() + for addition in additions: + key = canonical_typed_value(addition["value"]) + if key in seen_additions: + raise duplicate_value_label + seen_additions.add(key) + if key in index: + result[index[key]] = addition + else: + index[key] = len(result) + result.append(addition) + return result +``` + +This is specification pseudocode, not an implementation API. Typed values use +their exact canonical identity; numeric positive zero canonicalization and +string exactness remain inherited. Emit one complete `replace_value_labels` +operation per variable in group/source order. A preceding `VALUE LABELS` +replaces state; a following `ADD VALUE LABELS` merges against that replacement, +not against the original request. + +- [ ] **Step 3: Prove Plan 0.1 output selection.** + +At least four successful additive-label cases contain no Plan 0.2 operation and +must declare: + +```json +"expected_plan_contract": "openstatspec-transformation-plan-v0.1" +``` + +Their expected plans contain only complete ordered `replace_value_labels` +operations and validate against `plan-0.1.schema.json`. + +- [ ] **Step 4: Run the final manifest inventory and identity checks.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: Plan `4/26`, Frontend declared `13/44/35`, Frontend effective +`13/44/90`, Binding `6/11`; all hashes and references validate. + +- [ ] **Step 5: Commit additive-label fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py docs/spss-syntax-frontend-profile-0.3.md +git commit -m "spec: define additive value label syntax" +``` + +### Task 9: Complete examples, release notes, and adapter handoff + +**Files:** +- Create: `examples/spss-syntax-transformation-0.3.md` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `ROADMAP.md` +- Modify: `docs/spss-frontend-roadmap.md` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Consumes: complete 0.3 schema/profile/manifest and validated counts. +- Produces: specification release-candidate documentation and exact downstream adapter claim gates. + +- [ ] **Step 1: Add failing documentation presence checks.** + +```python +def test_frontend_03_release_candidate_documentation_is_complete() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") + example = (ROOT / "examples/spss-syntax-transformation-0.3.md").read_text(encoding="utf-8") + assert "spss-syntax-frontend-0.3.schema.json" in readme + assert "SPSS Syntax Frontend 0.3" in changelog + assert "Frontend 0.3" in roadmap + assert "ADD VALUE LABELS" in example + assert "Plan 0.1/0.2" in example +``` + +- [ ] **Step 2: Write one complete 0.3 example.** + +Use one exact program whose corresponding manifest cases already prove each +production independently: + +```spss +* Normalize the selected score block. +RECODE score_a TO score_c (LOWEST THRU 0 = 0) (1 THRU HIGHEST = 1). +ADD VALUE LABELS score_a TO score_c 0 'Non-positive' 1 'Positive'. +FORMATS score_a TO score_c (F1.0). +IF (NOT (score_a = score_b)) mismatch = 1. +EXECUTE. +``` + +The example must state that `mismatch` is an existing numeric target, explain +dictionary-order `TO`, finite open-range lowering, complete additive-label +replacement, `< OR >` lowering, source hashing with comments, Plan 0.1/0.2 +selection, and unchanged SQL/Dolt binding policy. + +- [ ] **Step 3: Update the artifact index and Unreleased changelog.** + +In `README.md`, list the 0.3 profile, schema, and conformance manifest beside +0.1/0.2. In `CHANGELOG.md` under Unreleased record: + +- Frontend 0.3 is a release candidate, not a published stable claim. +- It adds comments, existing-variable `TO`, grouped forms, predicate aliases and + bounded `NOT`, open recode ranges, and additive value labels. +- It emits only immutable Plan 0.1/0.2 and adds no SQL binding operation. +- Repository validation now covers all Plan/Frontend/Binding schemas, + manifests, references, and hashes. + +- [ ] **Step 4: Mark only completed roadmap milestones.** + +In `ROADMAP.md` section 7, mark the Frontend 0.3 specification item complete +only after schema, profile, 35 declared cases, inherited compatibility, and +repository validation are all present. Leave adapter implementation and all +later expression/schema/case/group milestones unchecked. + +In `docs/spss-frontend-roadmap.md`, mark Milestone 0 and Milestone 1 as +specification-complete while stating that adapter claims remain pending and +separately gated. + +- [ ] **Step 5: Add exact adapter handoff gates to the 0.3 profile.** + +Require an adapter claim to: + +1. pin the exact future specification release commit; +2. accept only the exact Frontend 0.3 request contract for this suite; +3. run all 90 effective Frontend 0.3 cases; +4. preserve every inherited Plan 0.1/0.2 object and hash; +5. pass all 35 declared cases with exact source/plan hashes and diagnostics; +6. continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites; +7. keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy unchanged. + +State explicitly that adapter evidence does not block specification publication. + +- [ ] **Step 6: Promote the 0.3 profile from draft to release candidate.** + +Only after every prior task is green, change its status line to: + +```markdown +Status: release candidate for the next OpenStatSpec minor release. This profile +is not published stable until a protected or signed specification tag targets +its exact reviewed commit and tag-context CI passes. +``` + +Do not choose the final specification release number in this implementation +task; release maintainers select it from the complete set of included normative +changes. + +- [ ] **Step 7: Run final specification verification.** + +Run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +git diff --check +git status --short +``` + +Expected: all pytest tests pass, validator reports Plan `4/26`, Frontend +declared `13/44/35`, Frontend effective `13/44/90`, Binding `6/11`, diff check +is clean, and status lists only intended release-candidate documentation. + +- [ ] **Step 8: Commit release-candidate documentation.** + +```bash +git add examples/spss-syntax-transformation-0.3.md README.md CHANGELOG.md ROADMAP.md docs/spss-frontend-roadmap.md docs/spss-syntax-frontend-profile-0.3.md src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "docs: prepare SPSS frontend 0.3 candidate" +``` + +### Task 10: Review and publish the specification branch + +**Files:** +- Verify: all files changed by Tasks 1-9 + +**Interfaces:** +- Consumes: complete specification candidate and green repository checks. +- Produces: reviewable branch/PR; no specification tag and no adapter claim. + +- [ ] **Step 1: Review the complete contract diff.** + +Run: + +```bash +git status --short --branch +git diff origin/main...HEAD --stat +git diff origin/main...HEAD -- transformation docs conformance src tests tools README.md CHANGELOG.md ROADMAP.md pyproject.toml .github/workflows/ci.yml +git log --oneline origin/main..HEAD +``` + +Confirm no published 0.1/0.2 schema or manifest changed by running: + +```bash +git diff --exit-code origin/main...HEAD -- \ + transformation/plan-0.1.schema.json \ + transformation/plan-0.2.schema.json \ + transformation/spss-syntax-frontend-0.1.schema.json \ + transformation/spss-syntax-frontend-0.2.schema.json \ + conformance/transformation-plan-0.1.json \ + conformance/transformation-plan-0.2.json \ + conformance/spss-syntax-frontend-0.1.json \ + conformance/spss-syntax-frontend-0.2.json \ + conformance/in-place-transformation-0.1.json \ + conformance/in-place-transformation-0.2.json +``` + +Expected: exit zero and no diff for every published artifact. + +- [ ] **Step 2: Run the final clean-checkout gate.** + +Create an isolated clean checkout of `HEAD`, install `.[dev]`, and run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: all checks pass without depending on sibling adapter repositories. + +- [ ] **Step 3: Push and create a specification PR.** + +```bash +git push -u origin agent/spss-frontend-03 +gh pr create --repo OpenStatSpec/specification \ + --base main \ + --head agent/spss-frontend-03 \ + --title "Define SPSS syntax frontend 0.3" \ + --body $'Defines the bounded SPSS Syntax Frontend 0.3 contract.\n\n- Emits only immutable Plan 0.1/0.2 objects.\n- Adds 35 declared and 90 effective frontend cases.\n- Preserves all published 0.1/0.2 artifacts and hashes.\n- Adds repository-owned schema, reference, and hash validation.\n\nVerification: `python3 -m pytest -q` and `python3 tools/validate_repository.py`.' +``` + +The PR body must list contract boundaries, exact declared/effective case counts, +immutable older-artifact verification, validator/test commands, and downstream +adapter gates. Do not create a specification tag or GitHub release in this task. From 22af9aadaca2a55ea5715efd13730437fdba979a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 13:54:14 +0300 Subject: [PATCH 03/15] test: validate contract artifact inventory --- .github/workflows/ci.yml | 2 + pyproject.toml | 3 +- src/openstatspec_specification/artifacts.py | 366 ++++++++++++++++++++ tests/test_contract_artifacts.py | 49 +++ 4 files changed, 419 insertions(+), 1 deletion(-) create mode 100644 src/openstatspec_specification/artifacts.py create mode 100644 tests/test_contract_artifacts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43474af..d14f43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,4 +19,6 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: "3.13" + - run: python -m pip install --disable-pip-version-check -e ".[dev]" + - run: python -m pytest - run: python tools/validate_repository.py diff --git a/pyproject.toml b/pyproject.toml index 54d5d8d..e4f844a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,9 +10,10 @@ readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] +dependencies = ["jsonschema>=4.23,<5"] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8,<9"] [tool.hatch.build.targets.wheel] packages = ["src/openstatspec_specification"] diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py new file mode 100644 index 0000000..49ccf2c --- /dev/null +++ b/src/openstatspec_specification/artifacts.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + + +PLAN_MANIFESTS = { + "0.1": "conformance/transformation-plan-0.1.json", + "0.2": "conformance/transformation-plan-0.2.json", +} +FRONTEND_MANIFESTS = { + "0.1": "conformance/spss-syntax-frontend-0.1.json", + "0.2": "conformance/spss-syntax-frontend-0.2.json", +} +BINDING_MANIFESTS = { + "0.1": "conformance/in-place-transformation-0.1.json", + "0.2": "conformance/in-place-transformation-0.2.json", +} + + +class ArtifactValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class ArtifactInventory: + plan_cases: Mapping[str, int] + frontend_declared_cases: Mapping[str, int] + frontend_effective_cases: Mapping[str, int] + binding_cases: Mapping[str, int] + + +def canonical_json_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def source_hash(source_text: str) -> str: + normalized = source_text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactValidationError(message) + + +def _canonical_parts(relative: str) -> list[str]: + _require( + isinstance(relative, str) + and not relative.startswith("/") + and "\\" not in relative, + f"noncanonical artifact path: {relative}", + ) + parts = relative.split("/") + _require( + all(part not in {"", ".", ".."} for part in parts), + f"noncanonical artifact path: {relative}", + ) + return parts + + +def _safe_candidate(root: Path, relative: str) -> Path: + parts = _canonical_parts(relative) + candidate = root.joinpath(*parts) + current = root + try: + for part in parts: + current = current / part + _require( + not current.is_symlink(), + f"artifact path traverses symlink: {relative}", + ) + except OSError as error: + raise ArtifactValidationError( + f"artifact path cannot be inspected: {relative} ({type(error).__name__})" + ) from None + return candidate + + +def _load_json(root: Path, relative: str) -> object: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + return json.loads(candidate.read_text(encoding="utf-8")) + except ArtifactValidationError: + raise + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + + +def _require_file(root: Path, relative: str) -> None: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + except OSError as error: + raise ArtifactValidationError( + f"artifact cannot be inspected: {relative} ({type(error).__name__})" + ) from None + + +def _resolve_manifest_reference(manifest_relative: str, reference: str) -> str: + _require( + isinstance(reference, str) + and bool(reference) + and not reference.startswith("/") + and "\\" not in reference, + f"noncanonical manifest reference: {reference!r}", + ) + resolved: list[str] = [] + candidate = PurePosixPath(manifest_relative).parent / reference + for part in candidate.parts: + if part in {"", "."}: + continue + if part == "..": + _require(bool(resolved), f"manifest reference escapes repository root: {reference}") + resolved.pop() + continue + resolved.append(part) + _require(bool(resolved), f"manifest reference resolves to repository root: {reference}") + return "/".join(resolved) + + +def _require_unique_case_ids(manifest: Mapping[str, object], context: str) -> None: + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{context}: cases must be a list") + seen: set[str] = set() + for case in cases: + _require(isinstance(case, dict), f"{context}: each case must be an object") + case_id = case.get("id") + _require( + isinstance(case_id, str) and bool(case_id), + f"{context}: case id must be a non-empty string", + ) + _require(case_id not in seen, f"{context}: duplicate case id: {case_id}") + seen.add(case_id) + + +def _schema(root: Path, relative: str) -> Mapping[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: schema must be an object") + try: + Draft202012Validator.check_schema(value) + except SchemaError as error: + raise ArtifactValidationError( + f"{relative}: invalid JSON Schema: {error.message}" + ) from None + return value + + +def _load_manifest(root: Path, relative: str, version: str, contract: str) -> dict[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: manifest must be an object") + _require( + value.get("manifest_version") == version, + f"{relative}: manifest_version must be {version}", + ) + _require( + value.get("contract") == contract, + f"{relative}: contract must be {contract}", + ) + _require_unique_case_ids(value, relative) + return value + + +def _validate_cases( + cases: object, + schema: Mapping[str, object], + field: str, + context: str, +) -> None: + _require(isinstance(cases, list), f"{context}: cases must be a list") + validator = Draft202012Validator(schema) + for index, case in enumerate(cases): + _require(isinstance(case, dict), f"{context}: each case must be an object") + value = case.get(field) + try: + validator.validate(value) + except ValidationError as error: + raise ArtifactValidationError( + f"{context}.cases[{index}].{field}: schema validation failed: {error.message}" + ) from None + + +def _validate_expected_plans( + cases: object, + schemas: Mapping[str, Mapping[str, object]], + context: str, +) -> None: + _require(isinstance(cases, list), f"{context}: cases must be a list") + for index, case in enumerate(cases): + _require(isinstance(case, dict), f"{context}: each case must be an object") + for field in ("expected_plan", "expected_plan_0_1"): + if field not in case: + continue + plan = case[field] + _require(isinstance(plan, dict), f"{context}.cases[{index}].{field}: must be an object") + contract = plan.get("contract") + _require( + isinstance(contract, str) and contract in schemas, + f"{context}.cases[{index}].{field}: unsupported plan contract", + ) + try: + Draft202012Validator(schemas[contract]).validate(plan) + except ValidationError as error: + raise ArtifactValidationError( + f"{context}.cases[{index}].{field}: schema validation failed: {error.message}" + ) from None + + +def _require_exact_field(value: object, expected: object, context: str) -> None: + _require(value == expected, f"{context}: has an unexpected value") + + +def validate_contract_artifacts(root: Path) -> ArtifactInventory: + plan_schemas: dict[str, Mapping[str, object]] = {} + plan_manifests: dict[str, dict[str, object]] = {} + plan_contracts = { + "0.1": "openstatspec-transformation-plan-v0.1", + "0.2": "openstatspec-transformation-plan-v0.2", + } + for version, manifest_relative in PLAN_MANIFESTS.items(): + manifest = _load_manifest(root, manifest_relative, version, plan_contracts[version]) + schema_reference = { + "0.1": "../transformation/plan-0.1.schema.json", + "0.2": "../transformation/plan-0.2.schema.json", + }[version] + _require_exact_field( + manifest.get("schema"), + schema_reference, + f"{manifest_relative}: schema", + ) + schema_relative = _resolve_manifest_reference(manifest_relative, schema_reference) + plan_schemas[version] = _schema(root, schema_relative) + _validate_cases(manifest.get("cases"), plan_schemas[version], "plan", manifest_relative) + plan_manifests[version] = manifest + + frontend_schemas: dict[str, Mapping[str, object]] = {} + frontend_manifests: dict[str, dict[str, object]] = {} + frontend_contracts = { + "0.1": "openstatspec-spss-syntax-frontend-v0.1", + "0.2": "openstatspec-spss-syntax-frontend-v0.2", + } + for version, manifest_relative in FRONTEND_MANIFESTS.items(): + manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) + request_reference = { + "0.1": "../transformation/spss-syntax-frontend-0.1.schema.json", + "0.2": "../transformation/spss-syntax-frontend-0.2.schema.json", + }[version] + _require_exact_field( + manifest.get("request_schema"), + request_reference, + f"{manifest_relative}: request_schema", + ) + request_relative = _resolve_manifest_reference(manifest_relative, request_reference) + frontend_schemas[version] = _schema(root, request_relative) + _validate_cases( + manifest.get("cases"), + frontend_schemas[version], + "request", + manifest_relative, + ) + + if version == "0.1": + _require_exact_field( + manifest.get("plan_contract"), + "openstatspec-transformation-plan-v0.1", + f"{manifest_relative}: plan_contract", + ) + plan_reference = "../transformation/plan-0.1.schema.json" + _require_exact_field( + manifest.get("plan_schema"), + plan_reference, + f"{manifest_relative}: plan_schema", + ) + resolved_plan_reference = _resolve_manifest_reference( + manifest_relative, + plan_reference, + ) + plan_schema = _schema(root, resolved_plan_reference) + _validate_expected_plans( + manifest.get("cases"), + {"openstatspec-transformation-plan-v0.1": plan_schema}, + manifest_relative, + ) + else: + _require_exact_field( + manifest.get("plan_contracts"), + [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2", + ], + f"{manifest_relative}: plan_contracts", + ) + plan_references = { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json", + } + _require_exact_field( + manifest.get("plan_schemas"), + plan_references, + f"{manifest_relative}: plan_schemas", + ) + resolved_plan_schemas = { + contract: _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + for contract, reference in plan_references.items() + } + _validate_expected_plans( + manifest.get("cases"), + resolved_plan_schemas, + manifest_relative, + ) + frontend_manifests[version] = manifest + + binding_contracts = { + "0.1": "openstatspec-in-place-transformation-v0.1", + "0.2": "openstatspec-in-place-transformation-v0.2", + } + binding_manifests: dict[str, dict[str, object]] = {} + for version, manifest_relative in BINDING_MANIFESTS.items(): + manifest = _load_manifest(root, manifest_relative, version, binding_contracts[version]) + if version == "0.2": + audit_reference = "../sql/transformation-plan-profile-schema.sql" + _require_exact_field( + manifest.get("audit_schema"), + audit_reference, + f"{manifest_relative}: audit_schema", + ) + resolved_audit_reference = _resolve_manifest_reference( + manifest_relative, + audit_reference, + ) + _require_file(root, resolved_audit_reference) + binding_manifests[version] = manifest + + return ArtifactInventory( + plan_cases={version: len(manifest["cases"]) for version, manifest in plan_manifests.items()}, + frontend_declared_cases={ + version: len(manifest["cases"]) + for version, manifest in frontend_manifests.items() + }, + frontend_effective_cases={ + version: len(manifest["cases"]) + for version, manifest in frontend_manifests.items() + }, + binding_cases={ + version: len(manifest["cases"]) + for version, manifest in binding_manifests.items() + }, + ) diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py new file mode 100644 index 0000000..bddd2cb --- /dev/null +++ b/tests/test_contract_artifacts.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest + +from openstatspec_specification.artifacts import ArtifactValidationError +from openstatspec_specification.artifacts import validate_contract_artifacts + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_released_contract_artifact_inventory_is_valid() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.plan_cases == {"0.1": 4, "0.2": 26} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44} + assert inventory.binding_cases == {"0.1": 6, "0.2": 11} + + +def copied_artifacts(tmp_path: Path) -> Path: + for directory in ("conformance", "transformation", "sql", "docs"): + shutil.copytree(ROOT / directory, tmp_path / directory) + shutil.copy2(ROOT / "CHANGELOG.md", tmp_path / "CHANGELOG.md") + shutil.copy2(ROOT / "ROADMAP.md", tmp_path / "ROADMAP.md") + return tmp_path + + +def test_duplicate_manifest_case_id_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"].append(manifest["cases"][0]) + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="duplicate case id"): + validate_contract_artifacts(root) + + +def test_invalid_json_schema_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "transformation/plan-0.1.schema.json" + schema = json.loads(path.read_text(encoding="utf-8")) + schema["type"] = "not-a-json-schema-type" + path.write_text(json.dumps(schema), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="invalid JSON Schema"): + validate_contract_artifacts(root) From f062d4e0a35b28341e5e88591b20926b428937a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:07:28 +0300 Subject: [PATCH 04/15] test: verify transformation artifact identity --- src/openstatspec_specification/artifacts.py | 388 ++++++++++++++++++-- tests/test_contract_artifacts.py | 39 ++ tools/validate_repository.py | 13 +- 3 files changed, 412 insertions(+), 28 deletions(-) diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 49ccf2c..112fbb5 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -48,7 +48,11 @@ def canonical_json_bytes(value: object) -> bytes: def source_hash(source_text: str) -> str: normalized = source_text.replace("\r\n", "\n").replace("\r", "\n") - return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + return _sha256(normalized.encode("utf-8")) + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() def _require(condition: bool, message: str) -> None: @@ -221,12 +225,354 @@ def _validate_expected_plans( ) from None +def _expected_error(case: Mapping[str, object], context: str) -> str | None: + value = case.get("expected_error") + _require( + value is None or (isinstance(value, str) and bool(value)), + f"{context}: expected_error must be null or a non-empty string", + ) + return value if isinstance(value, str) else None + + +def _schema_is_valid(value: object, schema: Mapping[str, object]) -> bool: + try: + Draft202012Validator(schema).validate(value) + except ValidationError: + return False + return True + + +def _validate_plan_manifests( + root: Path, + manifests: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, object]]: + """Return version -> case-id -> successful canonical plan case.""" + successful: dict[str, Mapping[str, object]] = {} + for version, manifest in manifests.items(): + manifest_relative = PLAN_MANIFESTS[version] + schema_reference = manifest.get("schema") + _require( + isinstance(schema_reference, str), + f"{manifest_relative}: schema must be a string", + ) + schema = _schema(root, _resolve_manifest_reference(manifest_relative, schema_reference)) + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + successful_cases: dict[str, Mapping[str, object]] = {} + validator = Draft202012Validator(schema) + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + expected_error = _expected_error(case, context) + plan = case.get("plan") + _require(isinstance(plan, dict), f"{context}.plan: must be an object") + expected_hash = case.get("expected_plan_hash") + if expected_error is not None: + _require( + "expected_plan_hash" not in case or expected_hash is None, + f"{context}: failing case must omit expected_plan_hash or set it to null", + ) + continue + + try: + validator.validate(plan) + except ValidationError as error: + raise ArtifactValidationError( + f"{context}.plan: schema validation failed: {error.message}" + ) from None + _require( + plan.get("contract") == manifest.get("contract"), + f"{context}.plan: contract does not match manifest", + ) + actual_hash = _sha256(canonical_json_bytes(plan)) + _require( + isinstance(expected_hash, str) and expected_hash == actual_hash, + f"{context}: plan hash mismatch", + ) + successful_cases[case["id"]] = case + successful[version] = successful_cases + return successful + + +def _frontend_plan_schemas( + root: Path, + version: str, + manifest: Mapping[str, object], +) -> Mapping[str, Mapping[str, object]]: + manifest_relative = FRONTEND_MANIFESTS[version] + if version == "0.1": + contract = manifest.get("plan_contract") + reference = manifest.get("plan_schema") + _require( + isinstance(contract, str) and isinstance(reference, str), + f"{manifest_relative}: plan contract and schema must be declared", + ) + return { + contract: _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + } + + references = manifest.get("plan_schemas") + _require(isinstance(references, dict), f"{manifest_relative}: plan_schemas must be an object") + schemas: dict[str, Mapping[str, object]] = {} + for contract, reference in references.items(): + _require( + isinstance(contract, str) and isinstance(reference, str), + f"{manifest_relative}: plan schema references must be strings", + ) + schemas[contract] = _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + return schemas + + +def _resolve_frontend_plans( + root: Path, + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, Mapping[str, object]]]: + """Return version -> case-id -> resolved canonical plan for successes.""" + resolved: dict[str, dict[str, Mapping[str, object]]] = {} + for version in ("0.1", "0.2"): + manifest = manifests[version] + manifest_relative = FRONTEND_MANIFESTS[version] + request_reference = manifest.get("request_schema") + _require( + isinstance(request_reference, str), + f"{manifest_relative}: request_schema must be a string", + ) + request_schema = _schema( + root, + _resolve_manifest_reference(manifest_relative, request_reference), + ) + plan_schemas = _frontend_plan_schemas(root, version, manifest) + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + successful: dict[str, Mapping[str, object]] = {} + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + expected_error = _expected_error(case, context) + request = case.get("request") + _require(isinstance(request, dict), f"{context}.request: must be an object") + source_text = request.get("source_text") + if isinstance(source_text, str): + actual_source_hash = source_hash(source_text) + _require( + isinstance(case.get("expected_source_hash"), str) + and case.get("expected_source_hash") == actual_source_hash, + f"{context}: source hash mismatch", + ) + request_valid = _schema_is_valid(request, request_schema) + if expected_error == "plan_schema_invalid": + _require( + not request_valid, + f"{context}: plan_schema_invalid requires an invalid request", + ) + else: + _require( + request_valid, + f"{context}.request: schema validation failed", + ) + + plan_fields = ( + "expected_plan", + "expected_plan_0_1", + "expected_plan_case", + "expected_plan_case_0_1", + ) + present_plan_fields = [field for field in plan_fields if field in case] + if expected_error is not None: + for field in (*plan_fields, "expected_plan_contract", "expected_plan_hash"): + _require( + field not in case, + f"{context}: failing case must omit {field}", + ) + continue + + _require( + len(present_plan_fields) == 1, + f"{context}: successful case must declare exactly one expected plan", + ) + expected_field = present_plan_fields[0] + plan: Mapping[str, object] + referenced_plan_hash: object = None + if expected_field in {"expected_plan", "expected_plan_0_1"}: + inline_plan = case.get(expected_field) + _require( + isinstance(inline_plan, dict), + f"{context}.{expected_field}: must be an object", + ) + plan = inline_plan + elif expected_field == "expected_plan_case": + plan_version = "0.1" if version == "0.1" else "0.2" + expected_contract = case.get("expected_plan_contract") + _require( + expected_contract is None + or expected_contract == manifests[version].get( + "plan_contract", "openstatspec-transformation-plan-v0.2" + ), + f"{context}: expected_plan_contract does not match expected_plan_case", + ) + reference = case.get(expected_field) + _require( + isinstance(reference, str), + f"{context}.{expected_field}: must be a string", + ) + referenced_case = plan_cases[plan_version].get(reference) + _require( + referenced_case is not None, + f"{context}: unknown plan case: {reference}", + ) + plan = referenced_case["plan"] + referenced_plan_hash = referenced_case.get("expected_plan_hash") + else: + _require( + version == "0.2", + f"{context}.{expected_field}: only Frontend 0.2 may use this reference", + ) + reference = case.get(expected_field) + _require( + isinstance(reference, str), + f"{context}.{expected_field}: must be a string", + ) + frontend_01_plan = resolved["0.1"].get(reference) + _require( + frontend_01_plan is not None, + f"{context}: unknown Frontend 0.1 plan case: {reference}", + ) + plan = frontend_01_plan + + contract = plan.get("contract") + _require( + isinstance(contract, str) and contract in plan_schemas, + f"{context}: unsupported expected plan contract", + ) + try: + Draft202012Validator(plan_schemas[contract]).validate(plan) + except ValidationError as error: + raise ArtifactValidationError( + f"{context}: expected plan schema validation failed: {error.message}" + ) from None + declared_contract = case.get("expected_plan_contract") + _require( + declared_contract is None or declared_contract == contract, + f"{context}: expected_plan_contract does not match plan", + ) + expected_hash = case.get("expected_plan_hash", referenced_plan_hash) + actual_hash = _sha256(canonical_json_bytes(plan)) + _require( + isinstance(expected_hash, str) and expected_hash == actual_hash, + f"{context}: plan hash mismatch", + ) + successful[case["id"]] = plan + resolved[version] = successful + return resolved + + +def _validate_binding_manifests( + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], + frontend_plans: Mapping[str, Mapping[str, Mapping[str, object]]], + frontend_manifests: Mapping[str, Mapping[str, object]], +) -> None: + for version, manifest in manifests.items(): + manifest_relative = BINDING_MANIFESTS[version] + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + expected_error = _expected_error(case, context) + if expected_error is not None: + _require( + case.get("mutation_started") is False + or ( + case.get("mutation_started") is True + and case.get("failure_point") == "after_data_and_metadata_before_audit" + ), + f"{context}: failing binding case must set mutation_started to false", + ) + if version != "0.2": + continue + + has_plan_reference = "applied_plan_case" in case + has_frontend_reference = "applied_frontend_case" in case + _require( + has_plan_reference == has_frontend_reference, + f"{context}: binding plan and frontend references must be paired", + ) + if not has_plan_reference: + continue + + plan_reference = case.get("applied_plan_case") + frontend_reference = case.get("applied_frontend_case") + _require( + isinstance(plan_reference, str), + f"{context}: applied_plan_case must be a string", + ) + _require( + isinstance(frontend_reference, str), + f"{context}: applied_frontend_case must be a string", + ) + plan_case = plan_cases["0.2"].get(plan_reference) + _require( + plan_case is not None, + f"{context}: unknown plan case: {plan_reference}", + ) + frontend_plan = frontend_plans["0.2"].get(frontend_reference) + _require( + frontend_plan is not None, + f"{context}: unknown frontend case: {frontend_reference}", + ) + _require( + frontend_plan == plan_case["plan"], + f"{context}: binding reference mismatch", + ) + + expected_audit = case.get("expected_audit") + if expected_audit is None: + continue + _require( + isinstance(expected_audit, dict), + f"{context}.expected_audit: must be an object", + ) + plan = plan_case["plan"] + frontend_case = next( + ( + frontend_case + for frontend_case in frontend_manifests["0.2"].get("cases", []) + if isinstance(frontend_case, dict) + and frontend_case.get("id") == frontend_reference + ), + None, + ) + _require( + isinstance(frontend_case, dict), + f"{context}: unknown frontend case: {frontend_reference}", + ) + expected_values = { + "plan_hash": plan_case.get("expected_plan_hash"), + "source_hash": frontend_case.get("expected_source_hash"), + "canonical_plan_json": canonical_json_bytes(plan).decode("utf-8"), + "operation_count": len(plan["operations"]), + } + for field, expected in expected_values.items(): + if field in expected_audit: + _require( + expected_audit[field] == expected, + f"{context}.expected_audit.{field}: audit identity mismatch", + ) + + def _require_exact_field(value: object, expected: object, context: str) -> None: _require(value == expected, f"{context}: has an unexpected value") def validate_contract_artifacts(root: Path) -> ArtifactInventory: - plan_schemas: dict[str, Mapping[str, object]] = {} plan_manifests: dict[str, dict[str, object]] = {} plan_contracts = { "0.1": "openstatspec-transformation-plan-v0.1", @@ -244,11 +590,10 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: f"{manifest_relative}: schema", ) schema_relative = _resolve_manifest_reference(manifest_relative, schema_reference) - plan_schemas[version] = _schema(root, schema_relative) - _validate_cases(manifest.get("cases"), plan_schemas[version], "plan", manifest_relative) + _schema(root, schema_relative) plan_manifests[version] = manifest + plan_cases = _validate_plan_manifests(root, plan_manifests) - frontend_schemas: dict[str, Mapping[str, object]] = {} frontend_manifests: dict[str, dict[str, object]] = {} frontend_contracts = { "0.1": "openstatspec-spss-syntax-frontend-v0.1", @@ -266,13 +611,7 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: f"{manifest_relative}: request_schema", ) request_relative = _resolve_manifest_reference(manifest_relative, request_reference) - frontend_schemas[version] = _schema(root, request_relative) - _validate_cases( - manifest.get("cases"), - frontend_schemas[version], - "request", - manifest_relative, - ) + _schema(root, request_relative) if version == "0.1": _require_exact_field( @@ -290,12 +629,7 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: manifest_relative, plan_reference, ) - plan_schema = _schema(root, resolved_plan_reference) - _validate_expected_plans( - manifest.get("cases"), - {"openstatspec-transformation-plan-v0.1": plan_schema}, - manifest_relative, - ) + _schema(root, resolved_plan_reference) else: _require_exact_field( manifest.get("plan_contracts"), @@ -314,19 +648,13 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: plan_references, f"{manifest_relative}: plan_schemas", ) - resolved_plan_schemas = { - contract: _schema( + for reference in plan_references.values(): + _schema( root, _resolve_manifest_reference(manifest_relative, reference), ) - for contract, reference in plan_references.items() - } - _validate_expected_plans( - manifest.get("cases"), - resolved_plan_schemas, - manifest_relative, - ) frontend_manifests[version] = manifest + frontend_plans = _resolve_frontend_plans(root, frontend_manifests, plan_cases) binding_contracts = { "0.1": "openstatspec-in-place-transformation-v0.1", @@ -348,6 +676,12 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: ) _require_file(root, resolved_audit_reference) binding_manifests[version] = manifest + _validate_binding_manifests( + binding_manifests, + plan_cases, + frontend_plans, + frontend_manifests, + ) return ArtifactInventory( plan_cases={version: len(manifest["cases"]) for version, manifest in plan_manifests.items()}, diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index bddd2cb..8dc148f 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -47,3 +47,42 @@ def test_invalid_json_schema_fails_closed(tmp_path: Path) -> None: path.write_text(json.dumps(schema), encoding="utf-8") with pytest.raises(ArtifactValidationError, match="invalid JSON Schema"): validate_contract_artifacts(root) + + +@pytest.mark.parametrize( + ("manifest_name", "field"), + [ + ("transformation-plan-0.2.json", "expected_plan_hash"), + ("spss-syntax-frontend-0.2.json", "expected_source_hash"), + ], +) +def test_hash_mismatch_fails_closed( + tmp_path: Path, manifest_name: str, field: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance" / manifest_name + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0][field] = "0" * 64 + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="hash mismatch"): + validate_contract_artifacts(root) + + +def test_missing_frontend_plan_reference_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["expected_plan_case"] = "absent-plan" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="unknown plan case"): + validate_contract_artifacts(root) + + +def test_binding_plan_and_frontend_references_must_agree(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/in-place-transformation-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["applied_plan_case"] = "strict-greater-than" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="binding reference mismatch"): + validate_contract_artifacts(root) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index e8808b6..20ffd24 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -13,6 +13,10 @@ DoltDeclarationError, main, ) +from openstatspec_specification.artifacts import ( # noqa: E402 + ArtifactValidationError, + validate_contract_artifacts, +) def require(condition: bool, message: str) -> None: @@ -140,6 +144,13 @@ def validate_repository_controls() -> None: if __name__ == "__main__": try: main() + inventory = validate_contract_artifacts(ROOT) validate_repository_controls() - except DoltDeclarationError as error: + print( + "Validated contract artifacts: " + f"Plan {dict(inventory.plan_cases)}, " + f"Frontend {dict(inventory.frontend_effective_cases)}, " + f"Binding {dict(inventory.binding_cases)}." + ) + except (DoltDeclarationError, ArtifactValidationError) as error: raise SystemExit(str(error)) from error From fb2b8773774758ae6dce5bcbea1062193b8ba74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:15:17 +0300 Subject: [PATCH 05/15] docs: align v0.3.0 release status --- CHANGELOG.md | 7 ++--- ROADMAP.md | 8 ++--- docs/spss-syntax-frontend-profile-0.2.md | 3 +- docs/transformation-plan-profile-0.2.md | 3 +- docs/transformation-plan-sql-binding-0.2.md | 3 +- src/openstatspec_specification/artifacts.py | 35 +++++++++++++++++++++ tests/test_contract_artifacts.py | 17 ++++++++++ 7 files changed, 64 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c0904..64e2820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,8 @@ Future changes will be listed here. ## v0.3.0 - 2026-08-12 -Release preparation for the planned stable `v0.3.0` tag. Until a protected or -signed `v0.3.0` tag targets this exact release commit, implementations MUST -identify it with `specification_status: release_candidate`, -`specification_release: null`, and the exact immutable specification commit. -The entries below become the stable release only when that tag exists. +OpenStatSpec `v0.3.0` is the stable public specification release at immutable +specification commit `cd8f198c68b849eb8ed018a894670a0904c2181d`. - Added backward-compatible Transformation Plan and SPSS Syntax Frontend 0.2 contracts for sequential numeric `COMPUTE` and conditional `IF`, bounded diff --git a/ROADMAP.md b/ROADMAP.md index 8d98fb2..bfba00c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -117,10 +117,10 @@ follows and does not block specification publication. record its `v0.3.0` release identifier; rebase conditional transformations on the lifecycle implementation and pass the combined service and conformance gates before making an adapter claim or package release. -5. [ ] Have PHP pin the exact specification commit and, once published, record - its `v0.3.0` release identifier; migrate to the canonical transformation plan - and pass the relevant conformance cases before making an adapter claim or - package release. +5. [x] PHP pinned specification `v0.3.0` commit + `cd8f198c68b849eb8ed018a894670a0904c2181d`, passed its complete PHP + 8.4/8.5 and PostgreSQL/MySQL/MariaDB/Dolt matrix, and published adapter + [v0.6.0](https://github.com/OpenStatSpec/php/releases/tag/v0.6.0). The pending in-place service matrix and adapter conformance work in sections 6 and 7 are downstream implementation gates. They do not block the `v0.3.0` diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index cab875c..2e05cfa 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -2,7 +2,8 @@ ## Status and boundary -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. This optional frontend maps a bounded SPSS-like command language to Transformation Plan 0.1 or 0.2. Its request contract is diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md index f3380fc..2340ee1 100644 --- a/docs/transformation-plan-profile-0.2.md +++ b/docs/transformation-plan-profile-0.2.md @@ -2,7 +2,8 @@ ## Status and scope -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. This optional profile is the backward-compatible successor to Transformation Plan 0.1. Its contract identifier is diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index b268c87..4c8c835 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -2,7 +2,8 @@ ## Status -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. ## Normative decision diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 112fbb5..a77b86d 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -22,6 +22,12 @@ "0.1": "conformance/in-place-transformation-0.1.json", "0.2": "conformance/in-place-transformation-0.2.json", } +V030_COMMIT = "cd8f198c68b849eb8ed018a894670a0904c2181d" +V030_PROFILE_DOCUMENTS = ( + "docs/transformation-plan-profile-0.2.md", + "docs/spss-syntax-frontend-profile-0.2.md", + "docs/transformation-plan-sql-binding-0.2.md", +) class ArtifactValidationError(ValueError): @@ -572,6 +578,35 @@ def _require_exact_field(value: object, expected: object, context: str) -> None: _require(value == expected, f"{context}: has an unexpected value") +def validate_release_metadata(root: Path) -> None: + for relative in V030_PROFILE_DOCUMENTS: + text = (root / relative).read_text(encoding="utf-8") + _require( + "Status: released in OpenStatSpec `v0.3.0`" in text, + f"{relative}: release status is inconsistent with v0.3.0", + ) + _require( + "release candidate for the planned OpenStatSpec `v0.3.0`" not in text, + f"{relative}: stale release status remains", + ) + roadmap = (root / "ROADMAP.md").read_text(encoding="utf-8") + _require("`v0.3.0` is the" in roadmap, "ROADMAP release tag mismatch") + _require( + "current public specification release and is immutable" in roadmap, + "ROADMAP release status mismatch", + ) + _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") + + classification = (root / "docs/spss-frontend-roadmap.md").read_text(encoding="utf-8") + for phrase in ( + "Frontend 0.3, Plan 0.1/0.2", + "New Plan/Frontend/Binding generation", + "Separate case-transformation profile", + "Explicit non-goal", + ): + _require(phrase in classification, f"SPSS classification is missing {phrase}") + + def validate_contract_artifacts(root: Path) -> ArtifactInventory: plan_manifests: dict[str, dict[str, object]] = {} plan_contracts = { diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 8dc148f..737f8fc 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -7,6 +7,7 @@ import pytest from openstatspec_specification.artifacts import ArtifactValidationError +from openstatspec_specification.artifacts import validate_release_metadata from openstatspec_specification.artifacts import validate_contract_artifacts @@ -21,6 +22,22 @@ def test_released_contract_artifact_inventory_is_valid() -> None: assert inventory.binding_cases == {"0.1": 6, "0.2": 11} +def test_v030_profile_status_is_consistently_released() -> None: + validate_release_metadata(ROOT) + + +def test_release_candidate_wording_fails_for_published_profile(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + text = path.read_text(encoding="utf-8").replace( + "Status: released in OpenStatSpec `v0.3.0`", + "Status: release candidate for OpenStatSpec `v0.3.0`", + ) + path.write_text(text, encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="release status"): + validate_release_metadata(root) + + def copied_artifacts(tmp_path: Path) -> Path: for directory in ("conformance", "transformation", "sql", "docs"): shutil.copytree(ROOT / directory, tmp_path / directory) From 3f69e2d96cd3e7a3809545991e4b0788e5c218c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:15:47 +0300 Subject: [PATCH 06/15] docs: align v0.3.0 release status --- tools/validate_repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index 20ffd24..9f9e7cc 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -16,6 +16,7 @@ from openstatspec_specification.artifacts import ( # noqa: E402 ArtifactValidationError, validate_contract_artifacts, + validate_release_metadata, ) @@ -140,6 +141,8 @@ def validate_repository_controls() -> None: ): require(f"CREATE TABLE {table} (" in schema, f"Schema table is missing: {table}") + validate_release_metadata(ROOT) + if __name__ == "__main__": try: From 7fe71b74b2704a13feae86d4382f0e1bbcad4ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:22:59 +0300 Subject: [PATCH 07/15] feat: define SPSS frontend 0.3 contract --- docs/spss-syntax-frontend-profile-0.3.md | 130 ++++++++++++++++++ src/openstatspec_specification/artifacts.py | 10 +- tests/test_contract_artifacts.py | 22 ++- .../spss-syntax-frontend-0.3.schema.json | 57 ++++++++ 4 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 docs/spss-syntax-frontend-profile-0.3.md create mode 100644 transformation/spss-syntax-frontend-0.3.schema.json diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md new file mode 100644 index 0000000..de254ee --- /dev/null +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -0,0 +1,130 @@ +# OpenStatSpec SPSS-like Syntax Frontend Profile 0.3 + +## Status and boundary + +This optional frontend profile maps a deliberately bounded SPSS-like command +language to Transformation Plan 0.1 or 0.2. Its request contract is +`openstatspec-spss-syntax-frontend-v0.3`. It is not full IBM SPSS Statistics +syntax and MUST reject every command, expression, or coercion not defined here. + +The request conforms to +[`../transformation/spss-syntax-frontend-0.3.schema.json`](../transformation/spss-syntax-frontend-0.3.schema.json). +All Frontend 0.2 lexical rules, command termination, exact source hashing, +case-insensitive name resolution, quote handling, and failure atomicity remain +in force. The input schema retains only ordered variables, storage kind, +variable and value labels, format family, width, decimals, and measurement +level. It does not declare physical identifiers. + +## Compatibility and output-plan selection + +Frontend 0.3 consumes unchanged Frontend 0.2 input-schema fields and unchanged +Plan 0.1 and Plan 0.2 contracts. Comments, `TO`, grouped syntax, open-ended +RECODE ranges, and additive value labels remain in the Plan 0.1 output subset. +Predicate aliases and `NOT` require the existing Plan 0.2 `conditional_assign` +syntax. A program containing only commands and productions in the Plan 0.1 +subset MUST emit an exact Plan 0.1 object. A program containing a 0.2-only +predicate production MUST emit Plan 0.2. No command is dropped, and the +frontend performs no data mutation, SQL execution, branch operation, or Dolt +commit. + +## Comments and source identity + +Leading-star comments are recognized only when `*` is the first non-whitespace +token at a command boundary. Such a comment ends at the next command period. +The `COMMENT` command is a command comment and ends at its command period. +Non-nested `/* ... */` comments are accepted wherever whitespace is legal. +Unterminated or nested block comments MUST return `spss_syntax_error`. + +Comments are retained in the LF-normalized source used for source hashing and +emit no operation. A comment-only program MUST return `spss_syntax_error` +because a valid plan cannot have zero operations. Source hashing remains +SHA-256 of UTF-8 source after CRLF and CR are converted to LF. The complete +emitted plan is hashed independently under its declared plan contract. + +## Existing-variable lists + +The bounded grammar is: + +```text +existing-variable-list := existing-variable + ((existing-variable | TO existing-variable))* +comparison-operator := "=" | "<" | "<=" | ">" | ">=" + | "NE" | "<>" | "~=" +predicate := comparison | "(" predicate ")" + | NOT predicate + | predicate AND predicate + | predicate OR predicate +open-range := LOWEST THRU finite-number + | finite-number THRU HIGHEST + | LOWEST THRU HIGHEST +``` + +`TO` expands inclusively using the current ordered input-schema state, not +variable spelling or a numeric suffix. Reverse ranges MUST return +`invalid_variable_range`; absent endpoints MUST return `unknown_variable`. +`TO` is forbidden for generated `INTO` target sequences. Resolved variables +retain their catalog spelling and source order; each occurrence is resolved at +the point its command is lowered. + +## Predicate aliases and NOT lowering + +`NE`, `<>`, and `~=` are aliases for ordered `< OR >` comparisons. `NOT` is +accepted only over the existing comparison and boolean grammar. It lowers +recursively using comparison complements and De Morgan's laws, while +preserving SQL three-valued truth. The complements are `=` to `< OR >`, `<` to +`>=`, `<=` to `>`, `>` to `<=`, and `>=` to `<`; `NE`, `<>`, and `~=` first +lower to `< OR >`. Boolean negation swaps `AND` and `OR` recursively. + +Canonical lowering flattens maximal same-operator boolean expressions while +preserving operand order. Parentheses override precedence across different +operators, but parentheses around a same-operator expression do not prevent +required flattening. Missing comparisons remain UNKNOWN and no lowering may +turn UNKNOWN into TRUE or FALSE. + +## Open-ended RECODE ranges + +`RECODE` retains the Frontend 0.2 typed selector and result rules. In addition, +numeric selectors may use the open-range forms in the grammar above. `LOWEST` +is the finite binary64 value with bits `ffefffffffffffff`, and `HIGHEST` is the +finite binary64 value with bits `7fefffffffffffff`. These bounds include every +finite binary64 value in the corresponding direction. SQL NULL/system missing +is not included in an open range. + +Open ranges are inclusive and lower to the existing ordered `recode` range +representation. Non-finite values, malformed finite numbers, and reverse +ranges are rejected with `spss_syntax_error` or `invalid_variable_range` as +applicable. The finite-number token remains the exact ASCII grammar inherited +from Frontend 0.2 and is interpreted as an exact base-ten value before +ties-to-even binary64 conversion. + +## ADD VALUE LABELS + +`ADD VALUE LABELS variable... value 'label' [value 'label' ...] +[/ variable... value 'label' ...].` resolves each existing variable against +the current typed schema. An existing typed value is updated at its current +ordinal. A new typed value is appended in source order. Duplicate values in a +single source group are invalid unless the operation is the update of that +same existing typed value according to this rule. + +The frontend emits one complete `replace_value_labels` operation per variable, +in source variable order, preserving all existing labels and applying the +updates and appended values. Variable and value types MUST agree; no implicit +numeric/string coercion is performed. The command remains in the Plan 0.1 +output subset. + +## Failure contract + +The frontend MUST produce no partial plan after any error. In addition to the +inherited 0.1 and 0.2 diagnostics, stable diagnostics include +`invalid_variable_range`, `unknown_variable`, and `spss_syntax_error` for the +rules defined here. Unsupported commands or productions return +`unsupported_spss_command` or `spss_syntax_error` as appropriate. Diagnostics +identify source spans but MUST NOT include credentials or unrelated row values. + +## Conformance + +Conformance cases for this profile are published in +[`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json) +when the Frontend 0.3 manifest is released. Until that manifest exists, the +schema and this profile are staged artifacts and no conformance case is +declared. diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index a77b86d..1443252 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -17,6 +17,7 @@ FRONTEND_MANIFESTS = { "0.1": "conformance/spss-syntax-frontend-0.1.json", "0.2": "conformance/spss-syntax-frontend-0.2.json", + "0.3": "conformance/spss-syntax-frontend-0.3.json", } BINDING_MANIFESTS = { "0.1": "conformance/in-place-transformation-0.1.json", @@ -633,13 +634,20 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: frontend_contracts = { "0.1": "openstatspec-spss-syntax-frontend-v0.1", "0.2": "openstatspec-spss-syntax-frontend-v0.2", + "0.3": "openstatspec-spss-syntax-frontend-v0.3", } for version, manifest_relative in FRONTEND_MANIFESTS.items(): - manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) request_reference = { "0.1": "../transformation/spss-syntax-frontend-0.1.schema.json", "0.2": "../transformation/spss-syntax-frontend-0.2.schema.json", + "0.3": "../transformation/spss-syntax-frontend-0.3.schema.json", }[version] + request_relative = _resolve_manifest_reference(manifest_relative, request_reference) + if version == "0.3" and not _safe_candidate(root, manifest_relative).is_file(): + _schema(root, request_relative) + frontend_manifests[version] = {"cases": []} + continue + manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) _require_exact_field( manifest.get("request_schema"), request_reference, diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 737f8fc..973bc60 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -17,11 +17,29 @@ def test_released_contract_artifact_inventory_is_valid() -> None: inventory = validate_contract_artifacts(ROOT) assert inventory.plan_cases == {"0.1": 4, "0.2": 26} - assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44} - assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 0} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 0} assert inventory.binding_cases == {"0.1": 6, "0.2": 11} +def test_frontend_03_request_schema_is_registered() -> None: + inventory = validate_contract_artifacts(ROOT) + assert "0.3" in inventory.frontend_declared_cases + + +def test_frontend_03_schema_changes_only_contract_identity() -> None: + old = json.loads((ROOT / "transformation/spss-syntax-frontend-0.2.schema.json").read_text()) + new = json.loads((ROOT / "transformation/spss-syntax-frontend-0.3.schema.json").read_text()) + assert new["$id"].endswith("spss-syntax-frontend-0.3.schema.json") + assert new["title"] == "OpenStatSpec SPSS-like Syntax Frontend Request 0.3" + assert new["properties"]["contract"]["const"] == "openstatspec-spss-syntax-frontend-v0.3" + for document in (old, new): + document.pop("$id") + document.pop("title") + document["properties"]["contract"].pop("const") + assert new == old + + def test_v030_profile_status_is_consistently_released() -> None: validate_release_metadata(ROOT) diff --git a/transformation/spss-syntax-frontend-0.3.schema.json b/transformation/spss-syntax-frontend-0.3.schema.json new file mode 100644 index 0000000..5523006 --- /dev/null +++ b/transformation/spss-syntax-frontend-0.3.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openstatspec.org/schemas/spss-syntax-frontend-0.3.schema.json", + "title": "OpenStatSpec SPSS-like Syntax Frontend Request 0.3", + "type": "object", + "additionalProperties": false, + "required": ["contract", "input_alias", "input_schema", "source_text"], + "properties": { + "contract": {"const": "openstatspec-spss-syntax-frontend-v0.3"}, + "input_alias": {"type": "string", "minLength": 1}, + "input_schema": { + "type": "object", "additionalProperties": false, "required": ["variables"], + "properties": { + "variables": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, "required": ["name", "storage_kind"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "storage_kind": {"enum": ["numeric", "string"]}, + "variable_label": {"type": ["string", "null"]}, + "value_labels": {"type": "array", "items": {"$ref": "#/$defs/value_label"}}, + "format_family": {"type": ["string", "null"]}, + "width": {"type": ["integer", "null"], "minimum": 1}, + "decimals": {"type": ["integer", "null"], "minimum": 0}, + "measurement_level": {"enum": ["nominal", "ordinal", "scale", null]} + } + } + } + } + }, + "source_text": {"type": "string", "minLength": 1} + }, + "$defs": { + "binary64_value": { + "type": "object", "additionalProperties": false, "required": ["type", "bits"], + "properties": { + "type": {"const": "binary64"}, + "bits": {"type": "string", "pattern": "^(?!8000000000000000$)(?!(?:7ff|fff)[0-9a-f]{13}$)[0-9a-f]{16}$"} + } + }, + "string_value": { + "type": "object", "additionalProperties": false, "required": ["type", "value"], + "properties": {"type": {"const": "string"}, "value": {"type": "string"}} + }, + "typed_value": {"oneOf": [{"$ref": "#/$defs/binary64_value"}, {"$ref": "#/$defs/string_value"}]}, + "value_label": { + "type": "object", + "additionalProperties": false, + "required": ["value", "label"], + "properties": { + "value": {"$ref": "#/$defs/typed_value"}, + "label": {"type": "string"} + } + } + } +} From 54186059e9c9a6ed102e243bed22cfd3ce8c3c00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:28:29 +0300 Subject: [PATCH 08/15] fix: reject malformed frontend 0.3 manifest paths --- src/openstatspec_specification/artifacts.py | 16 ++++++++++++---- tests/test_contract_artifacts.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 1443252..630bea0 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -643,10 +643,18 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: "0.3": "../transformation/spss-syntax-frontend-0.3.schema.json", }[version] request_relative = _resolve_manifest_reference(manifest_relative, request_reference) - if version == "0.3" and not _safe_candidate(root, manifest_relative).is_file(): - _schema(root, request_relative) - frontend_manifests[version] = {"cases": []} - continue + if version == "0.3": + manifest_path = _safe_candidate(root, manifest_relative) + try: + manifest_path.stat() + except FileNotFoundError: + _schema(root, request_relative) + frontend_manifests[version] = {"cases": []} + continue + except OSError as error: + raise ArtifactValidationError( + f"artifact cannot be inspected: {manifest_relative} ({type(error).__name__})" + ) from None manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) _require_exact_field( manifest.get("request_schema"), diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 973bc60..a17554f 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import shutil from pathlib import Path @@ -40,6 +41,22 @@ def test_frontend_03_schema_changes_only_contract_identity() -> None: assert new == old +@pytest.mark.parametrize("artifact_type", ["directory", "fifo", "symlink"]) +def test_existing_frontend_03_manifest_artifact_type_fails_closed( + tmp_path: Path, artifact_type: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + if artifact_type == "directory": + path.mkdir() + elif artifact_type == "fifo": + os.mkfifo(path) + else: + path.symlink_to(root / "conformance") + with pytest.raises(ArtifactValidationError, match="artifact"): + validate_contract_artifacts(root) + + def test_v030_profile_status_is_consistently_released() -> None: validate_release_metadata(ROOT) From 74510a04d78789a23a9b63f43668c5c79518b6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:36:48 +0300 Subject: [PATCH 09/15] spec: define SPSS frontend 0.3 contract --- conformance/spss-syntax-frontend-0.3.json | 56 ++++++ src/openstatspec_specification/artifacts.py | 181 +++++++++++++++++--- tests/test_contract_artifacts.py | 21 ++- 3 files changed, 233 insertions(+), 25 deletions(-) create mode 100644 conformance/spss-syntax-frontend-0.3.json diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json new file mode 100644 index 0000000..a734307 --- /dev/null +++ b/conformance/spss-syntax-frontend-0.3.json @@ -0,0 +1,56 @@ +{ + "manifest_version": "0.3", + "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.3", + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "plan_contracts": [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2" + ], + "request_schema": "../transformation/spss-syntax-frontend-0.3.schema.json", + "plan_schemas": { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json" + }, + "inherited_manifests": [ + { + "manifest": "spss-syntax-frontend-0.1.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" + } + }, + { + "manifest": "spss-syntax-frontend-0.2.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": {} + } + ], + "cases": [ + { + "id": "reject-comment-only-program", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* comment." + }, + "expected_source_hash": "8bfac9af8d0884b68cebc65a42360871592dd6e4e9cbead0bdf7029b6b64f4fc", + "expected_error": "spss_syntax_error" + }, + { + "id": "inline-block-comment-is-ignored", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* comment */." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "3c4ef459e9a2021100ef166ef0038c5f920b126d916db4c7273537d21b256f65", + "expected_error": null + } + ] +} diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 630bea0..075499f 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -2,6 +2,7 @@ import hashlib import json +from copy import deepcopy from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Mapping @@ -336,14 +337,152 @@ def _frontend_plan_schemas( return schemas +def _version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def _expand_frontend_cases( + root: Path, + version: str, + manifest: Mapping[str, object], + manifests: Mapping[str, Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + """Return inherited cases with contract override followed by declared cases.""" + + def expand( + current_version: str, + current_manifest: Mapping[str, object], + stack: tuple[str, ...], + ) -> tuple[Mapping[str, object], ...]: + _require( + current_version not in stack, + f"{FRONTEND_MANIFESTS[current_version]}: inherited manifest cycle", + ) + current_stack = (*stack, current_version) + current_relative = FRONTEND_MANIFESTS[current_version] + declared_cases = current_manifest.get("cases") + _require(isinstance(declared_cases, list), f"{current_relative}: cases must be a list") + declared_by_id = { + case["id"]: case + for case in declared_cases + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + + entries = current_manifest.get("inherited_manifests", []) + _require( + isinstance(entries, list), + f"{current_relative}: inherited_manifests must be a list", + ) + expanded: list[Mapping[str, object]] = [] + for index, entry in enumerate(entries): + context = f"{current_relative}.inherited_manifests[{index}]" + _require(isinstance(entry, dict), f"{context}: must be an object") + reference = entry.get("manifest") + try: + parts = _canonical_parts(reference) + except ArtifactValidationError: + raise ArtifactValidationError( + f"{context}: invalid inherited manifest: {reference!r}" + ) from None + _require( + len(parts) == 1 and parts[0] == reference, + f"{context}: invalid inherited manifest: {reference!r}", + ) + target_versions = [ + target_version + for target_version, target_relative in FRONTEND_MANIFESTS.items() + if target_relative == f"conformance/{reference}" + and _version_key(target_version) < _version_key(current_version) + ] + _require( + len(target_versions) == 1 and target_versions[0] in manifests, + f"{context}: inherited manifest must target an older registered Frontend manifest", + ) + target_version = target_versions[0] + target_manifest = manifests[target_version] + _require( + entry.get("request_contract_override") == current_manifest.get("contract"), + f"{context}: request_contract_override must match current manifest contract", + ) + superseded = entry.get("superseded_cases") + _require(isinstance(superseded, dict), f"{context}: superseded_cases must be an object") + target_declared = target_manifest.get("cases") + _require( + isinstance(target_declared, list), + f"{FRONTEND_MANIFESTS[target_version]}: cases must be a list", + ) + target_by_id = { + case["id"]: case + for case in target_declared + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + for old_id, replacement_id in superseded.items(): + _require( + isinstance(old_id, str) and isinstance(replacement_id, str), + f"{context}: superseded case IDs must be strings", + ) + _require( + old_id in target_by_id, + f"{context}: unknown inherited case: {old_id}", + ) + _require( + replacement_id in declared_by_id, + f"{context}: unknown replacement case: {replacement_id}", + ) + old_request = target_by_id[old_id].get("request") + replacement_request = declared_by_id[replacement_id].get("request") + _require( + isinstance(old_request, dict) + and isinstance(replacement_request, dict) + and old_request.get("source_text") == replacement_request.get("source_text"), + f"{context}: superseded case source text mismatch: {old_id}", + ) + + for inherited_case in expand(target_version, target_manifest, current_stack): + old_id = inherited_case["id"] + if old_id in superseded: + continue + case = deepcopy(inherited_case) + case["id"] = f"{target_version}/{old_id}" + request = case.get("request") + _require( + isinstance(request, dict), + f"{context}: inherited case request must be an object", + ) + request["contract"] = entry["request_contract_override"] + expanded.append(case) + + expanded.extend(deepcopy(case) for case in declared_cases) + seen: set[str] = set() + for case in expanded: + case_id = case.get("id") + _require( + isinstance(case_id, str) and case_id not in seen, + f"{current_relative}: duplicate expanded case id: {case_id}", + ) + seen.add(case_id) + return tuple(expanded) + + return expand(version, manifest, ()) + + +def _case_source_version(case_id: str, version: str) -> str: + if version == "0.3" and "/" in case_id: + source_version, _, _ = case_id.partition("/") + if source_version in {"0.1", "0.2"}: + return source_version + return version + + def _resolve_frontend_plans( root: Path, manifests: Mapping[str, Mapping[str, object]], plan_cases: Mapping[str, Mapping[str, object]], + effective_cases: Mapping[str, tuple[Mapping[str, object], ...]], ) -> Mapping[str, Mapping[str, Mapping[str, object]]]: """Return version -> case-id -> resolved canonical plan for successes.""" resolved: dict[str, dict[str, Mapping[str, object]]] = {} - for version in ("0.1", "0.2"): + for version in ("0.1", "0.2", "0.3"): manifest = manifests[version] manifest_relative = FRONTEND_MANIFESTS[version] request_reference = manifest.get("request_schema") @@ -356,12 +495,12 @@ def _resolve_frontend_plans( _resolve_manifest_reference(manifest_relative, request_reference), ) plan_schemas = _frontend_plan_schemas(root, version, manifest) - cases = manifest.get("cases") - _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + cases = effective_cases[version] successful: dict[str, Mapping[str, object]] = {} for index, case in enumerate(cases): context = f"{manifest_relative}.cases[{index}]" _require(isinstance(case, dict), f"{context}: each case must be an object") + source_version = _case_source_version(case["id"], version) expected_error = _expected_error(case, context) request = case.get("request") _require(isinstance(request, dict), f"{context}.request: must be an object") @@ -415,11 +554,11 @@ def _resolve_frontend_plans( ) plan = inline_plan elif expected_field == "expected_plan_case": - plan_version = "0.1" if version == "0.1" else "0.2" + plan_version = "0.1" if source_version == "0.1" else "0.2" expected_contract = case.get("expected_plan_contract") _require( expected_contract is None - or expected_contract == manifests[version].get( + or expected_contract == manifests[source_version].get( "plan_contract", "openstatspec-transformation-plan-v0.2" ), f"{context}: expected_plan_contract does not match expected_plan_case", @@ -438,8 +577,8 @@ def _resolve_frontend_plans( referenced_plan_hash = referenced_case.get("expected_plan_hash") else: _require( - version == "0.2", - f"{context}.{expected_field}: only Frontend 0.2 may use this reference", + source_version in {"0.2", "0.3"}, + f"{context}.{expected_field}: only Frontend 0.2 and 0.3 may use this reference", ) reference = case.get(expected_field) _require( @@ -642,19 +781,6 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: "0.2": "../transformation/spss-syntax-frontend-0.2.schema.json", "0.3": "../transformation/spss-syntax-frontend-0.3.schema.json", }[version] - request_relative = _resolve_manifest_reference(manifest_relative, request_reference) - if version == "0.3": - manifest_path = _safe_candidate(root, manifest_relative) - try: - manifest_path.stat() - except FileNotFoundError: - _schema(root, request_relative) - frontend_manifests[version] = {"cases": []} - continue - except OSError as error: - raise ArtifactValidationError( - f"artifact cannot be inspected: {manifest_relative} ({type(error).__name__})" - ) from None manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) _require_exact_field( manifest.get("request_schema"), @@ -705,7 +831,16 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: _resolve_manifest_reference(manifest_relative, reference), ) frontend_manifests[version] = manifest - frontend_plans = _resolve_frontend_plans(root, frontend_manifests, plan_cases) + frontend_effective_cases = { + version: _expand_frontend_cases(root, version, manifest, frontend_manifests) + for version, manifest in frontend_manifests.items() + } + frontend_plans = _resolve_frontend_plans( + root, + frontend_manifests, + plan_cases, + frontend_effective_cases, + ) binding_contracts = { "0.1": "openstatspec-in-place-transformation-v0.1", @@ -741,8 +876,8 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: for version, manifest in frontend_manifests.items() }, frontend_effective_cases={ - version: len(manifest["cases"]) - for version, manifest in frontend_manifests.items() + version: len(cases) + for version, cases in frontend_effective_cases.items() }, binding_cases={ version: len(manifest["cases"]) diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index a17554f..60a8110 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -18,8 +18,8 @@ def test_released_contract_artifact_inventory_is_valid() -> None: inventory = validate_contract_artifacts(ROOT) assert inventory.plan_cases == {"0.1": 4, "0.2": 26} - assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 0} - assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 0} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 2} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 57} assert inventory.binding_cases == {"0.1": 6, "0.2": 11} @@ -41,12 +41,29 @@ def test_frontend_03_schema_changes_only_contract_identity() -> None: assert new == old +def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 2 + assert inventory.frontend_effective_cases["0.3"] == 57 + + +def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["inherited_manifests"][0]["manifest"] = "../outside.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="inherited manifest"): + validate_contract_artifacts(root) + + @pytest.mark.parametrize("artifact_type", ["directory", "fifo", "symlink"]) def test_existing_frontend_03_manifest_artifact_type_fails_closed( tmp_path: Path, artifact_type: str, ) -> None: root = copied_artifacts(tmp_path) path = root / "conformance/spss-syntax-frontend-0.3.json" + path.unlink() if artifact_type == "directory": path.mkdir() elif artifact_type == "fifo": From 6fb55103e508533e1a5f16a2f4b9ba1f74626a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:47:08 +0300 Subject: [PATCH 10/15] spec: add frontend comments and variable lists --- conformance/spss-syntax-frontend-0.3.json | 237 ++++++++++++++++++++++ tests/test_contract_artifacts.py | 14 +- 2 files changed, 247 insertions(+), 4 deletions(-) diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json index a734307..8af2257 100644 --- a/conformance/spss-syntax-frontend-0.3.json +++ b/conformance/spss-syntax-frontend-0.3.json @@ -51,6 +51,243 @@ "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", "expected_source_hash": "3c4ef459e9a2021100ef166ef0038c5f920b126d916db4c7273537d21b256f65", "expected_error": null + }, + { + "id": "comment-command-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "COMMENT recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "5211e79a67d5f493b48a6907d365ade7b877d8351191c9924121af30f7865989", + "expected_error": null + }, + { + "id": "leading-star-comment-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "dddbaf0dd69f364ccc895d0d3b65daa63208edfd9b4e8e2112f915f1e73e5d70", + "expected_error": null + }, + { + "id": "comments-change-source-hash-not-plan-hash", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "COMMENT first comment.\n* second comment.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "8b0d8f677faa7e919c89ace749d0e45f89531fc6e22bfc796819c23081c1a5dc", + "expected_error": null + }, + { + "id": "reject-unterminated-inline-comment", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* unterminated." + }, + "expected_source_hash": "d675e34c7617e21d3b298bdc139d69fe2239fce0c82c03933da409c5a735f064", + "expected_error": "spss_syntax_error" + }, + { + "id": "recode-to-expands-dictionary-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "RECODE a TO c (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "recode", "source": "a", "target": "a", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}, + {"op": "recode", "source": "middle", "target": "middle", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}, + {"op": "recode", "source": "c", "target": "c", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}} + ] + }, + "expected_plan_hash": "81c8b98d48b8c14ad43153bc52960e088c05a23fff3a6f60d01e3898650f2009", + "expected_source_hash": "365b8cadd07b148d7bbee906ce66ae608d3d3fd69ec82f6549623f724893b4f9", + "expected_error": null + }, + { + "id": "value-labels-to-expands-dictionary-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "VALUE LABELS a TO c 0 'No' 1 'Yes'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "replace_value_labels", "variable": "a", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"op": "replace_value_labels", "variable": "middle", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"op": "replace_value_labels", "variable": "c", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]} + ] + }, + "expected_plan_hash": "d3edd16ab980d21b1af06812fabf7f3e71d85972de9cb7f1717afac992450de1", + "expected_source_hash": "aa8a8028ac720e82794523bff70335c64e971cd8df7385ea2f6c9f60205bdbc0", + "expected_error": null + }, + { + "id": "formats-grouped-to-expands-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS a TO c (F8.2)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "set_format", "variable": "a", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "middle", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "c", "family": "F", "width": 8, "decimals": 2} + ] + }, + "expected_plan_hash": "c148754d88165fe10c5a1cd4c050a41b4e4a2839c8dad89531e972b69989aa7e", + "expected_source_hash": "9884908d306d12b46a02dd19001dba2ba3fc45b54a8ef40659cb586d0271d0d0", + "expected_error": null + }, + { + "id": "variable-level-to-expands-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "VARIABLE LEVEL a TO c (NOMINAL)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "set_measurement_level", "variable": "a", "level": "nominal"}, + {"op": "set_measurement_level", "variable": "middle", "level": "nominal"}, + {"op": "set_measurement_level", "variable": "c", "level": "nominal"} + ] + }, + "expected_plan_hash": "d3232768fb06358ae4f9e71da635908a133e19391fdaea125126e7d686ac1084", + "expected_source_hash": "1b1c7a9373fac15b8466e18a9f04c500e19747eb5e82a2ac3dc7b9301b450d09", + "expected_error": null + }, + { + "id": "to-sees-variable-created-by-preceding-command", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "COMPUTE appended = a.\nFORMATS c TO appended (F8.2)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "assign", "target": "appended", "target_mode": "create", "value": {"kind": "variable", "variable": "a"}}, + {"op": "set_format", "variable": "c", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "outside", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "appended", "family": "F", "width": 8, "decimals": 2} + ] + }, + "expected_plan_hash": "97e447dee8786ba50661b8348ccfc2ec03d7d0f8d9f694daca0f3962dda6dd78", + "expected_source_hash": "819cd323cd9382e4340e4d4dcbbb47a5a155ccac674f2bfc616c7ca78e3c8cd5", + "expected_error": null + }, + { + "id": "reject-reversed-to-range", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS c TO a (F8.2)." + }, + "expected_source_hash": "b4a66bad42565222b32b559724ebefb029d427dfa49845d7e5ea1de05aad42f1", + "expected_error": "invalid_variable_range" + }, + { + "id": "reject-unknown-to-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS a TO absent (F8.2)." + }, + "expected_source_hash": "6153086c283f905e177f45c9208e73e118315a6f8b496cfced05058da28009dd", + "expected_error": "unknown_variable" + }, + { + "id": "reject-to-generated-into-targets", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "RECODE a (1 = 0) INTO target TO target2." + }, + "expected_source_hash": "c9b4ae4e9bfb7e25c8720f278d50669250fde123dffd4d743adaf18c91f83bd1", + "expected_error": "spss_syntax_error" } ] } diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 60a8110..6af4479 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -18,8 +18,8 @@ def test_released_contract_artifact_inventory_is_valid() -> None: inventory = validate_contract_artifacts(ROOT) assert inventory.plan_cases == {"0.1": 4, "0.2": 26} - assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 2} - assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 57} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 14} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 69} assert inventory.binding_cases == {"0.1": 6, "0.2": 11} @@ -43,8 +43,14 @@ def test_frontend_03_schema_changes_only_contract_identity() -> None: def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 2 - assert inventory.frontend_effective_cases["0.3"] == 57 + assert inventory.frontend_declared_cases["0.3"] == 14 + assert inventory.frontend_effective_cases["0.3"] == 69 + + +def test_frontend_03_comment_and_varlist_case_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 14 + assert inventory.frontend_effective_cases["0.3"] == 69 def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: From 6daf8fc697e7dc5b58d5f25aba9c0754a56e3e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 14:56:12 +0300 Subject: [PATCH 11/15] spec: add frontend predicates and open ranges --- conformance/spss-syntax-frontend-0.3.json | 261 ++++++++++++++++++++++ docs/spss-syntax-frontend-profile-0.3.md | 19 +- tests/test_contract_artifacts.py | 18 +- 3 files changed, 288 insertions(+), 10 deletions(-) diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json index 8af2257..bf42111 100644 --- a/conformance/spss-syntax-frontend-0.3.json +++ b/conformance/spss-syntax-frontend-0.3.json @@ -288,6 +288,267 @@ }, "expected_source_hash": "c9b4ae4e9bfb7e25c8720f278d50669250fde123dffd4d743adaf18c91f83bd1", "expected_error": "spss_syntax_error" + }, + { + "id": "ne-keyword-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a NE 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "e0292c5367e5b401224d4efd2a5acddaa05dbddb104cd37af5f0eb187f19c0aa", + "expected_error": null + }, + { + "id": "angle-not-equal-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a <> 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "343d43d13ed8b58623da9231054cf92005ea0048f1701bbd0dd2bf5822afc4c5", + "expected_error": null + }, + { + "id": "tilde-not-equal-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a ~= 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "9ba239c1692adb87db79b742e1f2aca74b002b1ed32466eb1366422c844d382b", + "expected_error": null + }, + { + "id": "not-equality-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (source_a = 1)) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "e309a67df9b587b81766ed35afa770e99a0debc9918352e39f5d7cdb9058ba3e", + "expected_error": null + }, + { + "id": "not-nested-and-lowers-by-de-morgan", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "source_b", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (source_a = 1 AND source_b = 1)) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "318eec6c5949cb7eb3b700727f00c0baee4cd521735b97443a207e42201082d6", + "expected_source_hash": "d6d61d8e9a27df8a0a48b8e88b8f7622cdb150a02f041449892d0a0b0c5311a4", + "expected_error": null + }, + { + "id": "double-not-retains-canonical-comparison", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (NOT (source_a = 1))) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "fc0fc2836fcdec4852c869986182051c6ec39d44ca78621e3471b4c85330e122", + "expected_source_hash": "de80a1e13be7c1193afe63f88269a70ca50af471a5943b9126582053bb1a02a4", + "expected_error": null + }, + { + "id": "reject-not-numeric-truthiness", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT source_a) target = 1." + }, + "expected_source_hash": "bf41ec574a901bc28462fc237edcdcb6bc1b88cc51af184d273c523951ca9518", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-chained-comparison", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "source_b", "storage_kind": "numeric"}, + {"name": "source_c", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a < source_b < source_c) target = 1." + }, + "expected_source_hash": "d5c8cab87b293826f708757132fcb629954ffdac6037cdd1f75f85b1400dab62", + "expected_error": "spss_syntax_error" + }, + { + "id": "recode-lowest-thru-upper", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (LOWEST THRU 1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "ffefffffffffffff"}, "upper": {"type": "binary64", "bits": "3ff0000000000000"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "e6b7fc9887b90f153067856771eaa3f8164a8e0212025f4c81923e70be6cec42", + "expected_source_hash": "7c9e0b869927dcda8a06e2b92f3c738902fabb93a126d2159583eeace55b3e3d", + "expected_error": null + }, + { + "id": "recode-lower-thru-highest", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (1 THRU HIGHEST = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "3ff0000000000000"}, "upper": {"type": "binary64", "bits": "7fefffffffffffff"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "ccfeaf45c8663888d2fd85a5c4808bbd8a314d1a2418554916a1f2746c0f2bea", + "expected_source_hash": "93b2464fb376be1c79ef49069b20252e40ec0d134b7b078148ee651da359acf1", + "expected_error": null + }, + { + "id": "recode-lowest-thru-highest", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (LOWEST THRU HIGHEST = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "ffefffffffffffff"}, "upper": {"type": "binary64", "bits": "7fefffffffffffff"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "6316b3fe33d13c034202c1472f1473d0658fb14538a1f5def8f83c9142c2adeb", + "expected_source_hash": "ac84fad5b5d0e29caef5e7e80ae83f5ee00404eca619a643840111d8571829a5", + "expected_error": null + }, + { + "id": "reject-string-open-range", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "string"}]}, + "source_text": "RECODE source_a (LOWEST THRU 1 = 'x')." + }, + "expected_source_hash": "cfd96fba97131220e5fef64551c8f4f746ff0019dd8e46174f3cfc301a4689ed", + "expected_error": "type_mismatch" + }, + { + "id": "reject-highest-as-lower-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (HIGHEST THRU 1 = 0)." + }, + "expected_source_hash": "44481c08e11c8ce299c3e1673e43047a438201cf7bcb2b0c9e3f38d99a00fbac", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-lowest-as-upper-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (1 THRU LOWEST = 0)." + }, + "expected_source_hash": "e9e9468f64c03bfd2822a365913399b3d5391f02f600e12606720b744a1d1e19", + "expected_error": "spss_syntax_error" } ] } diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md index de254ee..160476a 100644 --- a/docs/spss-syntax-frontend-profile-0.3.md +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -75,6 +75,18 @@ preserving SQL three-valued truth. The complements are `=` to `< OR >`, `<` to `>=`, `<=` to `>`, `>` to `<=`, and `>=` to `<`; `NE`, `<>`, and `~=` first lower to `< OR >`. Boolean negation swaps `AND` and `OR` recursively. +The canonical lowering table is: + +```text +NOT (left = right) -> (left < right OR left > right) +NOT (left < right) -> left >= right +NOT (left <= right) -> left > right +NOT (left > right) -> left <= right +NOT (left >= right) -> left < right +NOT (a AND b) -> NOT a OR NOT b +NOT (a OR b) -> NOT a AND NOT b +``` + Canonical lowering flattens maximal same-operator boolean expressions while preserving operand order. Parentheses override precedence across different operators, but parentheses around a same-operator expression do not prevent @@ -124,7 +136,6 @@ identify source spans but MUST NOT include credentials or unrelated row values. ## Conformance Conformance cases for this profile are published in -[`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json) -when the Frontend 0.3 manifest is released. Until that manifest exists, the -schema and this profile are staged artifacts and no conformance case is -declared. +[`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json). +The manifest declares 28 Frontend 0.3 cases and expands to 83 effective cases +after inherited cases and the two published comment supersessions are applied. diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 6af4479..a864f57 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -18,8 +18,8 @@ def test_released_contract_artifact_inventory_is_valid() -> None: inventory = validate_contract_artifacts(ROOT) assert inventory.plan_cases == {"0.1": 4, "0.2": 26} - assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 14} - assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 69} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 28} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 83} assert inventory.binding_cases == {"0.1": 6, "0.2": 11} @@ -28,6 +28,12 @@ def test_frontend_03_request_schema_is_registered() -> None: assert "0.3" in inventory.frontend_declared_cases +def test_frontend_03_predicate_and_open_range_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 28 + assert inventory.frontend_effective_cases["0.3"] == 83 + + def test_frontend_03_schema_changes_only_contract_identity() -> None: old = json.loads((ROOT / "transformation/spss-syntax-frontend-0.2.schema.json").read_text()) new = json.loads((ROOT / "transformation/spss-syntax-frontend-0.3.schema.json").read_text()) @@ -43,14 +49,14 @@ def test_frontend_03_schema_changes_only_contract_identity() -> None: def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 14 - assert inventory.frontend_effective_cases["0.3"] == 69 + assert inventory.frontend_declared_cases["0.3"] == 28 + assert inventory.frontend_effective_cases["0.3"] == 83 def test_frontend_03_comment_and_varlist_case_inventory() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 14 - assert inventory.frontend_effective_cases["0.3"] == 69 + assert inventory.frontend_declared_cases["0.3"] == 28 + assert inventory.frontend_effective_cases["0.3"] == 83 def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: From f48f5b74fcb6b6cb949acd9e3d1f166e6344ea7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 15:06:21 +0300 Subject: [PATCH 12/15] spec: define additive value label syntax --- conformance/spss-syntax-frontend-0.3.json | 154 ++++++++++++++++++++++ docs/spss-syntax-frontend-profile-0.3.md | 11 +- tests/test_contract_artifacts.py | 22 ++-- 3 files changed, 174 insertions(+), 13 deletions(-) diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json index bf42111..f567c26 100644 --- a/conformance/spss-syntax-frontend-0.3.json +++ b/conformance/spss-syntax-frontend-0.3.json @@ -549,6 +549,160 @@ }, "expected_source_hash": "e9e9468f64c03bfd2822a365913399b3d5391f02f600e12606720b744a1d1e19", "expected_error": "spss_syntax_error" + }, + { + "id": "add-value-labels-preserves-existing-and-appends", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}]}, + "source_text": "ADD VALUE LABELS q1 2 'Maybe'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}, + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"} + ]}] + }, + "expected_plan_hash": "520ec60bb84b6ec93eb4d2df1b7c0ba8a02efdcc9527284fabcd9b3ec26e931b", + "expected_source_hash": "c386c3ba15782601cf86459ec089d724ae7de449ee168b0d1acd06e489604b66", + "expected_error": null + }, + { + "id": "add-value-labels-replaces-existing-label-in-place", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}]}, + "source_text": "ADD VALUE LABELS q1 0 'Zero'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Zero"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}] + }, + "expected_plan_hash": "0877ee65052235c31a57f76bafc4c647cee49bc75a43c89565a459ac86e33f7a", + "expected_source_hash": "b3899248d84f5229bb7ad410a748c209a5de6cce65dbc8c9c163e6cfc9d5a537", + "expected_error": null + }, + { + "id": "value-labels-then-add-value-labels-uses-ordered-state", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Original"} + ]}]}, + "source_text": "VALUE LABELS q1 5 'Five' 1 'One'.\nADD VALUE LABELS q1 2 'Two' 5 'Cinq'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Cinq"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"}, + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Two"} + ]}] + }, + "expected_plan_hash": "d28ccba13408620463b8770857acfa8de5ffa0ba59f9212214f8bbfbdd0fa088", + "expected_source_hash": "fafcf4a687aa9241bf70b6152d6d10e083e357f322c7008c5aea8c72c5ccfd12", + "expected_error": null + }, + { + "id": "add-value-labels-multiple-variable-group", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}]}, + {"name": "middle", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"name": "c", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"}]}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "ADD VALUE LABELS a TO c 3 'Other'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "replace_value_labels", "variable": "a", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]}, + {"op": "replace_value_labels", "variable": "middle", "labels": [ + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]}, + {"op": "replace_value_labels", "variable": "c", "labels": [ + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]} + ] + }, + "expected_plan_hash": "7cc86bf992d568cde74802df7e6e68bc18f79210e084659e81b01d63f36ab2f5", + "expected_source_hash": "3353a38e495a448cdaad8bbb0981e4883bd21c9698148e0f87ad7158d13ca629", + "expected_error": null + }, + { + "id": "add-string-value-labels", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "status", "storage_kind": "string", "value_labels": [ + {"value": {"type": "string", "value": "N"}, "label": "No"} + ]}]}, + "source_text": "ADD VALUE LABELS status 'Y' 'Yes'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "status", "labels": [ + {"value": {"type": "string", "value": "N"}, "label": "No"}, + {"value": {"type": "string", "value": "Y"}, "label": "Yes"} + ]}] + }, + "expected_plan_hash": "e53af9b03c09640d0b451879451eac5e25ccc7e8bf22836b8451650f6b790648", + "expected_source_hash": "a5cc8c9431e28c028f73874d985c7796733443baf1ac1832f533e39c81197b9e", + "expected_error": null + }, + { + "id": "reject-add-value-labels-duplicate-source-value", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "ADD VALUE LABELS q1 1 'One' 1 'Duplicate'." + }, + "expected_source_hash": "1b26095f77569a6d5e0ed785c777cd943850066e2075df642a24c834c9f49068", + "expected_error": "duplicate_value_label" + }, + { + "id": "reject-add-value-labels-type-mismatch", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "ADD VALUE LABELS q1 '1' 'One'." + }, + "expected_source_hash": "a0048e248ab54129fab1b6e3b528cf1ac220d1ab59a48a5547ae22b3eae45721", + "expected_error": "type_mismatch" } ] } diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md index 160476a..fa70439 100644 --- a/docs/spss-syntax-frontend-profile-0.3.md +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -115,14 +115,15 @@ ties-to-even binary64 conversion. [/ variable... value 'label' ...].` resolves each existing variable against the current typed schema. An existing typed value is updated at its current ordinal. A new typed value is appended in source order. Duplicate values in a -single source group are invalid unless the operation is the update of that -same existing typed value according to this rule. +single source group are invalid. Typed values use exact canonical identity, +including positive-zero numeric canonicalization and exact string contents. The frontend emits one complete `replace_value_labels` operation per variable, in source variable order, preserving all existing labels and applying the updates and appended values. Variable and value types MUST agree; no implicit -numeric/string coercion is performed. The command remains in the Plan 0.1 -output subset. +numeric/string coercion is performed. If a preceding `VALUE LABELS` command +replaced the label state, a following `ADD VALUE LABELS` command merges against +that replacement state. The command remains in the Plan 0.1 output subset. ## Failure contract @@ -137,5 +138,5 @@ identify source spans but MUST NOT include credentials or unrelated row values. Conformance cases for this profile are published in [`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json). -The manifest declares 28 Frontend 0.3 cases and expands to 83 effective cases +The manifest declares 35 Frontend 0.3 cases and expands to 90 effective cases after inherited cases and the two published comment supersessions are applied. diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index a864f57..485b402 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -15,11 +15,17 @@ ROOT = Path(__file__).resolve().parents[1] +def test_frontend_03_final_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} + + def test_released_contract_artifact_inventory_is_valid() -> None: inventory = validate_contract_artifacts(ROOT) assert inventory.plan_cases == {"0.1": 4, "0.2": 26} - assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 28} - assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 83} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} assert inventory.binding_cases == {"0.1": 6, "0.2": 11} @@ -30,8 +36,8 @@ def test_frontend_03_request_schema_is_registered() -> None: def test_frontend_03_predicate_and_open_range_inventory() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 28 - assert inventory.frontend_effective_cases["0.3"] == 83 + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 def test_frontend_03_schema_changes_only_contract_identity() -> None: @@ -49,14 +55,14 @@ def test_frontend_03_schema_changes_only_contract_identity() -> None: def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 28 - assert inventory.frontend_effective_cases["0.3"] == 83 + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 def test_frontend_03_comment_and_varlist_case_inventory() -> None: inventory = validate_contract_artifacts(ROOT) - assert inventory.frontend_declared_cases["0.3"] == 28 - assert inventory.frontend_effective_cases["0.3"] == 83 + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: From eba29fce8b78575f8ba617fce2c5e536182eabd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 15:15:52 +0300 Subject: [PATCH 13/15] docs: prepare SPSS frontend 0.3 candidate --- CHANGELOG.md | 9 ++++- README.md | 14 ++++++- ROADMAP.md | 8 ++-- docs/spss-frontend-roadmap.md | 14 +++++++ docs/spss-syntax-frontend-profile-0.3.md | 20 ++++++++++ examples/spss-syntax-transformation-0.3.md | 41 +++++++++++++++++++++ src/openstatspec_specification/artifacts.py | 26 +++++++++++++ tests/test_contract_artifacts.py | 12 ++++++ 8 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 examples/spss-syntax-transformation-0.3.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 64e2820..eabca06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,14 @@ ## Unreleased -Future changes will be listed here. +- SPSS Syntax Frontend 0.3 is a release candidate, not a published stable + claim. +- Added comments, existing-variable `TO`, grouped supported forms, predicate + aliases and bounded `NOT`, open recode ranges, and additive value labels. +- Frontend 0.3 emits only immutable Plan 0.1/0.2 objects and adds no SQL + binding operation. +- Repository validation now covers all Plan/Frontend/Binding schemas, + manifests, references, and hashes. ## v0.3.0 - 2026-08-12 diff --git a/README.md b/README.md index c631ae9..ea4d6fa 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,18 @@ to be provisioned separately before an in-place transformation apply. - `transformation/plan-0.1.schema.json` — canonical transformation-plan schema. - `sql/transformation-plan-profile-schema.sql` — compact in-place apply audit; it is not a dataset-version catalog. -- `conformance/transformation-plan-0.2.json` and - `conformance/spss-syntax-frontend-0.2.json` — additive plan and frontend +- `docs/spss-syntax-frontend-profile-0.1.md`, + `docs/spss-syntax-frontend-profile-0.2.md`, and + `docs/spss-syntax-frontend-profile-0.3.md` — bounded syntax frontend + profiles and their contract boundaries. +- `transformation/spss-syntax-frontend-0.1.schema.json`, + `transformation/spss-syntax-frontend-0.2.schema.json`, and + `transformation/spss-syntax-frontend-0.3.schema.json` — frontend request + schemas. +- `conformance/transformation-plan-0.2.json`, + `conformance/spss-syntax-frontend-0.1.json`, + `conformance/spss-syntax-frontend-0.2.json`, and + `conformance/spss-syntax-frontend-0.3.json` — additive plan and frontend conformance cases with independent golden hashes; `conformance/in-place-transformation-0.2.json` fixes the same-dataset/same-table execution invariants and the additional controlled Dolt context. The 0.1 schemas and fixtures remain unchanged. diff --git a/ROADMAP.md b/ROADMAP.md index bfba00c..85f8a7a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -68,9 +68,11 @@ Prerequisite: the SPSS profile, fixtures, and reference-adapter lessons are matu - [ ] Expand the bounded SPSS frontend through the independently versioned milestones in the [SPSS frontend roadmap](docs/spss-frontend-roadmap.md), without creating a general SPSS compatibility claim. -- [ ] Publish Frontend 0.3 as syntax-only expansion over immutable Plan 0.1/0.2: - comments, dictionary-order variable lists, grouped supported forms, ordinary - not-equal/NOT predicates, open recode ranges, and additive value labels. +- [x] Complete the Frontend 0.3 specification candidate as a syntax-only + expansion over immutable Plan 0.1/0.2 after its schema, profile, 35 declared + cases, inherited compatibility, and repository validation are complete: + comments, dictionary-order variable lists, grouped supported forms, ordinary + not-equal/NOT predicates, open recode ranges, and additive value labels. - [ ] Define typed numeric expressions, deterministic numeric/date functions, missing-value predicates and metadata, additional numeric/date formats, and restricted conditional blocks in a new Plan/Frontend/In-Place Binding diff --git a/docs/spss-frontend-roadmap.md b/docs/spss-frontend-roadmap.md index bbdcd17..03d88c6 100644 --- a/docs/spss-frontend-roadmap.md +++ b/docs/spss-frontend-roadmap.md @@ -12,6 +12,10 @@ not advertise general IBM SPSS Statistics compatibility, forward source expressions into SQL, or weaken the same-dataset/same-table and caller-owned Dolt history rules. +Frontend 0.3 is specification-complete as a release candidate. Adapter claims +remain pending and separately gated; adapter evidence does not block +specification publication. + ## Command classification | Capability | Intended boundary | Reason | @@ -30,6 +34,8 @@ Dolt history rules. ## Milestone 0: integrity and classification +- [x] Specification-complete: Plan, Frontend, and In-Place schemas, manifests, + references, canonical bytes, and source/plan hashes are validated. - Add specification-owned validation for Plan, Frontend, and In-Place schemas, manifests, references, canonical bytes, and source/plan hashes. - Keep release status consistent across profile documents, roadmap, changelog, @@ -37,8 +43,13 @@ Dolt history rules. - Classify every proposed command as frontend-only, new Plan/Binding work, a separate profile, or an explicit non-goal before accepting syntax. +Adapter claims remain pending and separately gated; adapter evidence does not +block specification publication. + ## Milestone 1: Frontend 0.3 over Plan 0.1/0.2 +- [x] Specification-complete: the schema, profile, 35 declared cases, + inherited compatibility, and repository validation are present. Frontend 0.3 should add syntax depth without adding plan operations: - `COMMENT`, leading-star comments, and inline block comments; @@ -55,6 +66,9 @@ The manifest must list only Plan 0.1 and Plan 0.2 outputs and prove that all inherited programs retain their exact prior plans and hashes. No SQL binding or executor change belongs in this milestone. +Adapter claims remain pending and separately gated; adapter evidence does not +block specification publication. + ## Milestone 2: typed numeric expressions and metadata Define a new Plan, Frontend, and In-Place Binding generation for: diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md index fa70439..26bc552 100644 --- a/docs/spss-syntax-frontend-profile-0.3.md +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -2,6 +2,10 @@ ## Status and boundary +Status: release candidate for the next OpenStatSpec minor release. This profile +is not published stable until a protected or signed specification tag targets +its exact reviewed commit and tag-context CI passes. + This optional frontend profile maps a deliberately bounded SPSS-like command language to Transformation Plan 0.1 or 0.2. Its request contract is `openstatspec-spss-syntax-frontend-v0.3`. It is not full IBM SPSS Statistics @@ -140,3 +144,19 @@ Conformance cases for this profile are published in [`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json). The manifest declares 35 Frontend 0.3 cases and expands to 90 effective cases after inherited cases and the two published comment supersessions are applied. + +## Adapter claim gate + +An adapter claim for this profile MUST satisfy every gate below: + +1. Pin the exact future specification release commit. +2. Accept only the exact Frontend 0.3 request contract for this suite. +3. Run all 90 effective Frontend 0.3 cases. +4. Preserve every inherited Plan 0.1/0.2 object and hash. +5. Pass all 35 declared cases with exact source/plan hashes and diagnostics. +6. Continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites. +7. Keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy + unchanged. + +Adapter evidence does not block specification publication. The specification +release-candidate gate and each downstream adapter claim gate are separate. diff --git a/examples/spss-syntax-transformation-0.3.md b/examples/spss-syntax-transformation-0.3.md new file mode 100644 index 0000000..ff6802e --- /dev/null +++ b/examples/spss-syntax-transformation-0.3.md @@ -0,0 +1,41 @@ +# SPSS-like transformation frontend 0.3 release-candidate example + +This exact program combines the Frontend 0.3 syntax additions. The input +schema has `score_a`, `score_b`, and `score_c` as numeric variables in that +dictionary order. `mismatch` is an existing numeric target; the program does +not create it. + +```spss +* Normalize the selected score block. +RECODE score_a TO score_c (LOWEST THRU 0 = 0) (1 THRU HIGHEST = 1). +ADD VALUE LABELS score_a TO score_c 0 'Non-positive' 1 'Positive'. +FORMATS score_a TO score_c (F1.0). +IF (NOT (score_a = score_b)) mismatch = 1. +EXECUTE. +``` + +`TO` expands inclusively in dictionary order, so each command addresses +`score_a`, `score_b`, and `score_c`, not names inferred from a suffix. The +finite open ranges lower `LOWEST` and `HIGHEST` to the finite binary64 +endpoints defined by the profile; SQL NULL/system-missing remains outside the +range. `ADD VALUE LABELS` lowers to a complete additive-label replacement for +each variable, preserving the ordered existing label state and applying the +updates. `NOT (score_a = score_b)` lowers to the existing `< OR >` comparison +form, with SQL three-valued truth preserved. + +The leading comment is retained in the LF-normalized source used for the +source SHA-256 hash, while comments emit no operation. The full emitted plan +has its own canonical plan hash. This program uses the Plan 0.1 subset for the +recode, labels, and formats, but the conditional assignment and lowered +predicate selects between the immutable Plan 0.1/0.2 outputs. A program without a Plan 0.2-only +predicate production selects Plan 0.1 instead. The corresponding manifest +cases prove each production independently, including +`recode-lowest-thru-upper`, `recode-lower-thru-highest`, +`add-value-labels-multiple-variable-group`, `formats-grouped-to-expands-source-order`, +and `not-equality-lowers-to-less-or-greater`. + +Frontend 0.3 adds no SQL binding operation. The existing same-dataset and +same-physical-table policy is unchanged. MySQL, MariaDB, and Dolt still +require caller-owned pre-provisioning for a new target, and Dolt still keeps +branch, HEAD, working-set, and commit ownership with the caller; the frontend +does not call `DOLT_COMMIT`. diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 075499f..836727b 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -30,6 +30,7 @@ "docs/spss-syntax-frontend-profile-0.2.md", "docs/transformation-plan-sql-binding-0.2.md", ) +FRONTEND_03_PROFILE_DOCUMENT = "docs/spss-syntax-frontend-profile-0.3.md" class ArtifactValidationError(ValueError): @@ -737,12 +738,37 @@ def validate_release_metadata(root: Path) -> None: ) _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") + frontend_03 = (root / FRONTEND_03_PROFILE_DOCUMENT).read_text(encoding="utf-8") + _require( + "Status: release candidate for the next OpenStatSpec minor release." in frontend_03 + and "is not published stable until a protected or signed specification tag targets" in frontend_03 + and "its exact reviewed commit and tag-context CI passes." in frontend_03, + f"{FRONTEND_03_PROFILE_DOCUMENT}: release-candidate status is missing", + ) + for phrase in ( + "Pin the exact future specification release commit.", + "Accept only the exact Frontend 0.3 request contract for this suite.", + "Run all 90 effective Frontend 0.3 cases.", + "Preserve every inherited Plan 0.1/0.2 object and hash.", + "Pass all 35 declared cases with exact source/plan hashes and diagnostics.", + "Continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites.", + "Keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy", + "Adapter evidence does not block specification publication.", + ): + _require( + phrase in frontend_03, + f"{FRONTEND_03_PROFILE_DOCUMENT}: adapter claim gate is missing {phrase}", + ) + classification = (root / "docs/spss-frontend-roadmap.md").read_text(encoding="utf-8") for phrase in ( "Frontend 0.3, Plan 0.1/0.2", "New Plan/Frontend/Binding generation", "Separate case-transformation profile", "Explicit non-goal", + "Milestone 0: integrity and classification\n\n- [x] Specification-complete", + "Milestone 1: Frontend 0.3 over Plan 0.1/0.2\n\n- [x] Specification-complete", + "Adapter claims remain pending and separately gated", ): _require(phrase in classification, f"SPSS classification is missing {phrase}") diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 485b402..3e314b1 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -65,6 +65,18 @@ def test_frontend_03_comment_and_varlist_case_inventory() -> None: assert inventory.frontend_effective_cases["0.3"] == 90 +def test_frontend_03_release_candidate_documentation_is_complete() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") + example = (ROOT / "examples/spss-syntax-transformation-0.3.md").read_text(encoding="utf-8") + assert "spss-syntax-frontend-0.3.schema.json" in readme + assert "SPSS Syntax Frontend 0.3" in changelog + assert "Frontend 0.3" in roadmap + assert "ADD VALUE LABELS" in example + assert "Plan 0.1/0.2" in example + + def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: root = copied_artifacts(tmp_path) path = root / "conformance/spss-syntax-frontend-0.3.json" From 7047dab7b8f00ed0754ea8a55a04c7c226826f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 15:30:08 +0300 Subject: [PATCH 14/15] fix: restore PHP adapter evidence gate --- .../task-10-report.md | 24 +++++++++++++++++++ ROADMAP.md | 6 ++--- 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 .superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md diff --git a/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md b/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md new file mode 100644 index 0000000..672a9bf --- /dev/null +++ b/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md @@ -0,0 +1,24 @@ +# Task 10 Fix Report + +## Scope + +Removed the completed PHP adapter v0.6.0 evidence claim from +`ROADMAP.md`. Restored section 9 item 5 as an unchecked downstream gate requiring +PHP to pin the exact specification commit and pass applicable conformance +evidence before changing adapter or package-release claims. + +No Frontend 0.3 documentation, published 0.1/0.2 artifact, manifest count, +validator behavior, or runtime code was changed. + +## Verification + +- `/tmp/opencode/specification-test-venv/bin/python -m pytest -q`: `51 passed`. +- `/tmp/opencode/specification-test-venv/bin/python tools/validate_repository.py`: + passed; reported Plan `4/26`, Frontend `13/44/90`, and Binding `6/11`. +- `git diff --exit-code --` for all published 0.1/0.2 schemas and manifests: + passed with no output. +- `git diff --check`: passed; Git emitted only the existing LF-to-CRLF warning. + +## Concerns + +- None identified. diff --git a/ROADMAP.md b/ROADMAP.md index 85f8a7a..e8ebe49 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -119,10 +119,8 @@ follows and does not block specification publication. record its `v0.3.0` release identifier; rebase conditional transformations on the lifecycle implementation and pass the combined service and conformance gates before making an adapter claim or package release. -5. [x] PHP pinned specification `v0.3.0` commit - `cd8f198c68b849eb8ed018a894670a0904c2181d`, passed its complete PHP - 8.4/8.5 and PostgreSQL/MySQL/MariaDB/Dolt matrix, and published adapter - [v0.6.0](https://github.com/OpenStatSpec/php/releases/tag/v0.6.0). +5. [ ] Have PHP pin the exact specification commit and pass applicable + conformance evidence before changing adapter or package-release claims. The pending in-place service matrix and adapter conformance work in sections 6 and 7 are downstream implementation gates. They do not block the `v0.3.0` From 14f35eb1bf688e503e9d3251671a88323e359bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?T=C3=B5nis=20Ormisson?= Date: Fri, 21 Aug 2026 20:18:29 +0300 Subject: [PATCH 15/15] fix: address SPSS frontend review findings --- .../task-10-report.md | 24 ---- README.md | 11 +- conformance/spss-syntax-frontend-0.3.json | 18 ++- docs/spss-frontend-roadmap.md | 9 +- docs/spss-syntax-frontend-profile-0.3.md | 18 +-- .../plans/2026-08-21-spss-frontend-03.md | 32 +++-- ...26-08-21-spss-frontend-expansion-design.md | 7 +- examples/spss-syntax-transformation-0.3.md | 10 +- src/openstatspec_specification/artifacts.py | 132 ++++++++---------- tests/test_contract_artifacts.py | 84 +++++++++++ 10 files changed, 214 insertions(+), 131 deletions(-) delete mode 100644 .superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md diff --git a/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md b/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md deleted file mode 100644 index 672a9bf..0000000 --- a/.superpowers/sdd/2026-08-21-spss-frontend-03/task-10-report.md +++ /dev/null @@ -1,24 +0,0 @@ -# Task 10 Fix Report - -## Scope - -Removed the completed PHP adapter v0.6.0 evidence claim from -`ROADMAP.md`. Restored section 9 item 5 as an unchecked downstream gate requiring -PHP to pin the exact specification commit and pass applicable conformance -evidence before changing adapter or package-release claims. - -No Frontend 0.3 documentation, published 0.1/0.2 artifact, manifest count, -validator behavior, or runtime code was changed. - -## Verification - -- `/tmp/opencode/specification-test-venv/bin/python -m pytest -q`: `51 passed`. -- `/tmp/opencode/specification-test-venv/bin/python tools/validate_repository.py`: - passed; reported Plan `4/26`, Frontend `13/44/90`, and Binding `6/11`. -- `git diff --exit-code --` for all published 0.1/0.2 schemas and manifests: - passed with no output. -- `git diff --check`: passed; Git emitted only the existing LF-to-CRLF warning. - -## Concerns - -- None identified. diff --git a/README.md b/README.md index ea4d6fa..c93888e 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,11 @@ permits a transformation to mutate a core import. That separate derived-data profile is not used by the SPSS-like in-place frontend below. -The release-candidate [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) -adds sequential bounded numeric assignment and conditional assignment to the +The released [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) +was released in OpenStatSpec `v0.3.0` and adds sequential bounded numeric assignment and conditional assignment to the unchanged 0.1 operations. The -[SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) +[released SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) +was released in OpenStatSpec `v0.3.0` and lowers `COMPUTE`, `IF`, `FORMATS`, `VARIABLE LEVEL`, and `EXECUTE` alongside the 0.1 `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS` subset. Programs using only the 0.1 subset retain exact 0.1 plan identity and hash. The @@ -50,6 +51,10 @@ creates no derived dataset, data copy, or OpenStatSpec undo layer; Dolt-specific history and commits remain Dolt's. MySQL, MariaDB, and Dolt require a new target to be provisioned separately before an in-place transformation apply. +The release-candidate [SPSS Syntax Frontend Profile 0.3](docs/spss-syntax-frontend-profile-0.3.md) +adds syntax-only expansion over the released Plan 0.1/0.2 contracts and is not +published stable yet. + ## Repository layout - `sql/server-version-policy.md` — server release-series claims and exact reference-CI targets. diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json index f567c26..2b2711e 100644 --- a/conformance/spss-syntax-frontend-0.3.json +++ b/conformance/spss-syntax-frontend-0.3.json @@ -613,13 +613,19 @@ "expected_plan_0_1": { "contract": "openstatspec-transformation-plan-v0.1", "input_alias": "parent", - "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ - {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Cinq"}, - {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"}, - {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Two"} - ]}] + "operations": [ + {"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Five"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"} + ]}, + {"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Cinq"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"}, + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Two"} + ]} + ] }, - "expected_plan_hash": "d28ccba13408620463b8770857acfa8de5ffa0ba59f9212214f8bbfbdd0fa088", + "expected_plan_hash": "ed2c5ebaab637b6156f04f012bd99837e7377ac93e2919ec46e58fdc81cacd89", "expected_source_hash": "fafcf4a687aa9241bf70b6152d6d10e083e357f322c7008c5aea8c72c5ccfd12", "expected_error": null }, diff --git a/docs/spss-frontend-roadmap.md b/docs/spss-frontend-roadmap.md index 03d88c6..ec44993 100644 --- a/docs/spss-frontend-roadmap.md +++ b/docs/spss-frontend-roadmap.md @@ -20,7 +20,8 @@ specification publication. | Capability | Intended boundary | Reason | | --- | --- | --- | -| Comments, existing-variable `TO`, grouped supported forms | Frontend 0.3, Plan 0.1/0.2 | Compile-time syntax expansion only | +| Comments, existing-variable `TO`, grouped labels and recodes | Frontend 0.3, Plan 0.1 | Compile-time syntax expansion only | +| Grouped `FORMATS` and `VARIABLE LEVEL` | Frontend 0.3, Plan 0.2 | Expands to existing Plan 0.2 metadata operations | | `NOT`, `NE`, `<>`, `~=` over the existing predicate grammar | Frontend 0.3, Plan 0.2 | Canonically lowers to existing comparisons and booleans | | `LOWEST`/`HIGHEST` recode ranges | Frontend 0.3, Plan 0.1/0.2 | Lowers to finite binary64 endpoints; system missing remains distinct | | `ADD VALUE LABELS` | Frontend 0.3, Plan 0.1/0.2 | Lowers ordered metadata state to complete label replacement | @@ -36,7 +37,7 @@ specification publication. - [x] Specification-complete: Plan, Frontend, and In-Place schemas, manifests, references, canonical bytes, and source/plan hashes are validated. -- Add specification-owned validation for Plan, Frontend, and In-Place schemas, +- [x] Add specification-owned validation for Plan, Frontend, and In-Place schemas, manifests, references, canonical bytes, and source/plan hashes. - Keep release status consistent across profile documents, roadmap, changelog, tags, and releases. @@ -50,11 +51,11 @@ block specification publication. - [x] Specification-complete: the schema, profile, 35 declared cases, inherited compatibility, and repository validation are present. -Frontend 0.3 should add syntax depth without adding plan operations: +Frontend 0.3 adds syntax depth without adding plan operations: - `COMMENT`, leading-star comments, and inline block comments; - existing-variable lists with dictionary-order `TO` expansion; -- grouped forms of supported labels, formats, levels, and recodes; +- grouped forms of supported labels and recodes (Plan 0.1), plus grouped `FORMATS` and `VARIABLE LEVEL` (Plan 0.2); - `NOT`, `NE`, `<>`, and `~=` over the existing predicate grammar with three-valued truth semantics; - `LOWEST` and `HIGHEST` recode ranges; and diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md index 26bc552..4cfa15e 100644 --- a/docs/spss-syntax-frontend-profile-0.3.md +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -22,14 +22,16 @@ level. It does not declare physical identifiers. ## Compatibility and output-plan selection Frontend 0.3 consumes unchanged Frontend 0.2 input-schema fields and unchanged -Plan 0.1 and Plan 0.2 contracts. Comments, `TO`, grouped syntax, open-ended -RECODE ranges, and additive value labels remain in the Plan 0.1 output subset. -Predicate aliases and `NOT` require the existing Plan 0.2 `conditional_assign` -syntax. A program containing only commands and productions in the Plan 0.1 -subset MUST emit an exact Plan 0.1 object. A program containing a 0.2-only -predicate production MUST emit Plan 0.2. No command is dropped, and the -frontend performs no data mutation, SQL execution, branch operation, or Dolt -commit. +Plan 0.1 and Plan 0.2 contracts. Comments, `TO`, grouped recodes and labels, +open-ended RECODE ranges, and additive value labels remain in the Plan 0.1 +output subset. Grouped `FORMATS` and `VARIABLE LEVEL` emit Plan 0.2 +`set_format` and `set_measurement_level` operations, respectively. Predicate +aliases and `NOT` require the existing Plan 0.2 `conditional_assign` syntax. +A program containing only commands and productions whose underlying operations +are in the Plan 0.1 subset MUST emit an exact Plan 0.1 object. A program +containing any Plan 0.2-only operation MUST emit Plan 0.2. No command is +dropped, and the frontend performs no data mutation, SQL execution, branch +operation, or Dolt commit. ## Comments and source identity diff --git a/docs/superpowers/plans/2026-08-21-spss-frontend-03.md b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md index 2aa1131..106c049 100644 --- a/docs/superpowers/plans/2026-08-21-spss-frontend-03.md +++ b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md @@ -363,8 +363,9 @@ For every Plan manifest: 1. Index cases by ID. 2. Require `expected_error` to be either `null` or a non-empty string. 3. For successful cases, require a `plan` object, validate it against the manifest's schema, calculate `sha256(canonical_json_bytes(plan))`, and require exact lowercase `expected_plan_hash` equality. -4. For failing cases, require `expected_plan_hash` to be absent or `null`; retain the invalid plan as declarative negative input and do not require it to satisfy the plan schema. -5. Require the plan's `contract` to match the manifest contract for successful cases. +4. For `plan_schema_invalid` cases, require a plan object that fails the selected Plan schema, with `expected_plan_hash` absent or `null`. +5. For semantic failures, require a plan object that passes the selected Plan schema, retain its non-empty `expected_error`, and omit only the success hash. +6. Require the plan's `contract` to match the manifest contract for successful plan objects; a schema-invalid negative may fail this contract constraint as part of its selected-schema failure. Add private helpers with these signatures: @@ -385,11 +386,11 @@ For each Frontend case: 1. Compute `source_hash(request["source_text"])` for every structurally readable request and require exact `expected_source_hash`. 2. Validate successful requests against `request_schema`. -3. For `plan_schema_invalid` failures, require request schema validation to fail; for other failures, require the request schema to pass. -4. Resolve successful expected plans from exactly one of `expected_plan`, `expected_plan_0_1`, `expected_plan_case`, or `expected_plan_case_0_1`. +3. Require every readable request to pass its request schema, including `plan_schema_invalid` cases; malformed request cases remain invalid requests and do not become plan-schema cases. +4. Resolve a successful or `plan_schema_invalid` expected plan from exactly one of `expected_plan`, `expected_plan_0_1`, `expected_plan_case`, or `expected_plan_case_0_1`. 5. Resolve `expected_plan_case` against the declared Plan manifest and `expected_plan_case_0_1` against Frontend 0.1's resolved expected plans. 6. Validate the resolved plan against the schema selected by its exact `contract`, recompute its canonical hash, and require equality with `expected_plan_hash`. -7. Require failing cases to omit expected plan objects, references, and hashes. +7. Require semantic frontend failures to omit expected plan objects, references, and hashes. A Frontend `plan_schema_invalid` case instead includes the invalid expected plan, selects a recognized Plan contract, validates the request structurally, and omits or nulls `expected_plan_hash`. Use this exact resolver signature: @@ -398,10 +399,19 @@ def _resolve_frontend_plans( root: Path, manifests: Mapping[str, Mapping[str, object]], plan_cases: Mapping[str, Mapping[str, object]], + effective_cases: Mapping[str, tuple[Mapping[str, object], ...]], ) -> Mapping[str, Mapping[str, Mapping[str, object]]]: """Return version -> case-id -> resolved canonical plan for successes.""" ``` +The fourth argument is the version-to-effective-case mapping produced by +`_expand_frontend_cases`; it includes inherited cases with namespaced IDs and +declared replacements, after superseded cases are removed. The resolver returns +only structurally valid successful plans. Semantic negative frontend cases keep +their `expected_error` and no plan fields. A `plan_schema_invalid` case has a +schema-valid request and one inline invalid plan field, with a recognized plan +contract and no expected plan hash. + - [ ] **Step 5: Implement In-Place reference and audit identity validation.** For In-Place 0.2 cases containing `applied_plan_case` and @@ -704,7 +714,7 @@ Normatively require: - `NOT` is accepted only over the existing comparison/boolean grammar and lowers recursively with comparison complements and De Morgan's laws, preserving SQL three-valued truth and canonical same-operator flattening; - open ranges use `ffefffffffffffff` for negative maximum finite binary64 and `7fefffffffffffff` for positive maximum finite binary64; SQL NULL/system missing is not included; - `ADD VALUE LABELS` updates an existing typed value at its current ordinal and appends new values in source order, then emits one complete `replace_value_labels` operation per variable; -- comments, `TO`, grouped syntax, open ranges, and additive labels remain in the Plan 0.1 output subset; predicate aliases/`NOT` require existing Plan 0.2 `conditional_assign` syntax. +- comments, `TO`, grouped recodes and labels, open ranges, and additive labels remain in the Plan 0.1 output subset; grouped `FORMATS`/`VARIABLE LEVEL` and predicate aliases/`NOT` use existing Plan 0.2 operations. - [ ] **Step 5: Register Frontend 0.3 in the validator with an empty draft manifest.** @@ -819,7 +829,6 @@ Add: ```python def _expand_frontend_cases( - root: Path, version: str, manifest: Mapping[str, object], manifests: Mapping[str, Mapping[str, object]], @@ -835,9 +844,12 @@ expected plan hash, and expected diagnostic remain unchanged. `superseded_cases` maps an inherited case ID to a declared 0.3 replacement case ID. Require every old ID to exist, every replacement ID to exist in declared -cases, and every replacement to use the same `source_text`. Exclude superseded -old cases from the effective set. Reject inheritance cycles and duplicate -expanded IDs. +cases, and the replacement request to be identical to the inherited request +except for the current request contract. This preserves `input_alias`, +`input_schema`, and `source_text`; the replacement may change the expected +diagnostic and expected plan fields to document the new behavior. Exclude +superseded old cases from the effective set. Reject inheritance cycles and +duplicate expanded IDs. - [ ] **Step 4: Run the focused tests and complete Task 4/5 commit.** diff --git a/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md index a2e8d14..445f224 100644 --- a/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md +++ b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md @@ -65,7 +65,12 @@ catalog mutation, audit behavior, rollback, or profile capability rules change. Older schemas, fixtures, canonical JSON, hashes, and diagnostics remain immutable. New manifests inherit successful and failing older cases and prove -that old source continues to emit the same old plans. +that old source continues to emit the same old plans. An inherited case is +identical in the effective set, apart from its namespaced ID and request +contract override, unless its ID appears in `superseded_cases`. A superseding +replacement must use the same request input alias, input schema, and source +text; it may change only the expected diagnostic and expected plan fields that +describe the revised frontend behavior. ## Milestones diff --git a/examples/spss-syntax-transformation-0.3.md b/examples/spss-syntax-transformation-0.3.md index ff6802e..9648389 100644 --- a/examples/spss-syntax-transformation-0.3.md +++ b/examples/spss-syntax-transformation-0.3.md @@ -25,11 +25,11 @@ form, with SQL three-valued truth preserved. The leading comment is retained in the LF-normalized source used for the source SHA-256 hash, while comments emit no operation. The full emitted plan -has its own canonical plan hash. This program uses the Plan 0.1 subset for the -recode, labels, and formats, but the conditional assignment and lowered -predicate selects between the immutable Plan 0.1/0.2 outputs. A program without a Plan 0.2-only -predicate production selects Plan 0.1 instead. The corresponding manifest -cases prove each production independently, including +has its own canonical plan hash over the immutable Plan 0.1/0.2 outputs. The recode and labels use the Plan 0.1 +operation subset, while grouped `FORMATS`, the conditional assignment, and its +lowered predicate use Plan 0.2 operations. A program without any Plan 0.2-only +operation selects Plan 0.1 instead. The corresponding manifest cases prove +each production independently, including `recode-lowest-thru-upper`, `recode-lower-thru-highest`, `add-value-labels-multiple-variable-group`, `formats-grouped-to-expands-source-order`, and `not-equality-lowers-to-less-or-greater`. diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py index 836727b..052ed39 100644 --- a/src/openstatspec_specification/artifacts.py +++ b/src/openstatspec_specification/artifacts.py @@ -115,6 +115,19 @@ def _load_json(root: Path, relative: str) -> object: ) from None +def _load_text(root: Path, relative: str) -> str: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + return candidate.read_text(encoding="utf-8") + except ArtifactValidationError: + raise + except (OSError, UnicodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + + def _require_file(root: Path, relative: str) -> None: candidate = _safe_candidate(root, relative) try: @@ -189,51 +202,6 @@ def _load_manifest(root: Path, relative: str, version: str, contract: str) -> di return value -def _validate_cases( - cases: object, - schema: Mapping[str, object], - field: str, - context: str, -) -> None: - _require(isinstance(cases, list), f"{context}: cases must be a list") - validator = Draft202012Validator(schema) - for index, case in enumerate(cases): - _require(isinstance(case, dict), f"{context}: each case must be an object") - value = case.get(field) - try: - validator.validate(value) - except ValidationError as error: - raise ArtifactValidationError( - f"{context}.cases[{index}].{field}: schema validation failed: {error.message}" - ) from None - - -def _validate_expected_plans( - cases: object, - schemas: Mapping[str, Mapping[str, object]], - context: str, -) -> None: - _require(isinstance(cases, list), f"{context}: cases must be a list") - for index, case in enumerate(cases): - _require(isinstance(case, dict), f"{context}: each case must be an object") - for field in ("expected_plan", "expected_plan_0_1"): - if field not in case: - continue - plan = case[field] - _require(isinstance(plan, dict), f"{context}.cases[{index}].{field}: must be an object") - contract = plan.get("contract") - _require( - isinstance(contract, str) and contract in schemas, - f"{context}.cases[{index}].{field}: unsupported plan contract", - ) - try: - Draft202012Validator(schemas[contract]).validate(plan) - except ValidationError as error: - raise ArtifactValidationError( - f"{context}.cases[{index}].{field}: schema validation failed: {error.message}" - ) from None - - def _expected_error(case: Mapping[str, object], context: str) -> str | None: value = case.get("expected_error") _require( @@ -281,6 +249,18 @@ def _validate_plan_manifests( "expected_plan_hash" not in case or expected_hash is None, f"{context}: failing case must omit expected_plan_hash or set it to null", ) + try: + validator.validate(plan) + except ValidationError: + _require( + expected_error == "plan_schema_invalid", + f"{context}: semantic failure must have a schema-valid plan", + ) + else: + _require( + expected_error != "plan_schema_invalid", + f"{context}: plan_schema_invalid requires a schema-invalid plan", + ) continue try: @@ -343,7 +323,6 @@ def _version_key(version: str) -> tuple[int, ...]: def _expand_frontend_cases( - root: Path, version: str, manifest: Mapping[str, object], manifests: Mapping[str, Mapping[str, object]], @@ -433,10 +412,14 @@ def expand( old_request = target_by_id[old_id].get("request") replacement_request = declared_by_id[replacement_id].get("request") _require( - isinstance(old_request, dict) - and isinstance(replacement_request, dict) - and old_request.get("source_text") == replacement_request.get("source_text"), - f"{context}: superseded case source text mismatch: {old_id}", + isinstance(old_request, dict) and isinstance(replacement_request, dict), + f"{context}: superseded case request mismatch: {old_id}", + ) + expected_replacement_request = deepcopy(old_request) + expected_replacement_request["contract"] = entry["request_contract_override"] + _require( + replacement_request == expected_replacement_request, + f"{context}: superseded case request mismatch: {old_id}", ) for inherited_case in expand(target_version, target_manifest, current_stack): @@ -468,9 +451,9 @@ def expand( def _case_source_version(case_id: str, version: str) -> str: - if version == "0.3" and "/" in case_id: + if "/" in case_id: source_version, _, _ = case_id.partition("/") - if source_version in {"0.1", "0.2"}: + if source_version in FRONTEND_MANIFESTS: return source_version return version @@ -514,16 +497,10 @@ def _resolve_frontend_plans( f"{context}: source hash mismatch", ) request_valid = _schema_is_valid(request, request_schema) - if expected_error == "plan_schema_invalid": - _require( - not request_valid, - f"{context}: plan_schema_invalid requires an invalid request", - ) - else: - _require( - request_valid, - f"{context}.request: schema validation failed", - ) + _require( + request_valid, + f"{context}.request: schema validation failed", + ) plan_fields = ( "expected_plan", @@ -532,7 +509,7 @@ def _resolve_frontend_plans( "expected_plan_case_0_1", ) present_plan_fields = [field for field in plan_fields if field in case] - if expected_error is not None: + if expected_error is not None and expected_error != "plan_schema_invalid": for field in (*plan_fields, "expected_plan_contract", "expected_plan_hash"): _require( field not in case, @@ -601,9 +578,19 @@ def _resolve_frontend_plans( try: Draft202012Validator(plan_schemas[contract]).validate(plan) except ValidationError as error: + if expected_error == "plan_schema_invalid": + _require( + case.get("expected_plan_hash") is None, + f"{context}: plan_schema_invalid must omit expected_plan_hash or set it to null", + ) + continue raise ArtifactValidationError( f"{context}: expected plan schema validation failed: {error.message}" ) from None + _require( + expected_error != "plan_schema_invalid", + f"{context}: plan_schema_invalid requires a schema-invalid plan", + ) declared_contract = case.get("expected_plan_contract") _require( declared_contract is None or declared_contract == contract, @@ -641,7 +628,7 @@ def _validate_binding_manifests( case.get("mutation_started") is True and case.get("failure_point") == "after_data_and_metadata_before_audit" ), - f"{context}: failing binding case must set mutation_started to false", + f"{context}: failing binding case must set mutation_started to false or declare failure_point=after_data_and_metadata_before_audit", ) if version != "0.2": continue @@ -721,7 +708,7 @@ def _require_exact_field(value: object, expected: object, context: str) -> None: def validate_release_metadata(root: Path) -> None: for relative in V030_PROFILE_DOCUMENTS: - text = (root / relative).read_text(encoding="utf-8") + text = " ".join(_load_text(root, relative).split()) _require( "Status: released in OpenStatSpec `v0.3.0`" in text, f"{relative}: release status is inconsistent with v0.3.0", @@ -730,7 +717,7 @@ def validate_release_metadata(root: Path) -> None: "release candidate for the planned OpenStatSpec `v0.3.0`" not in text, f"{relative}: stale release status remains", ) - roadmap = (root / "ROADMAP.md").read_text(encoding="utf-8") + roadmap = " ".join(_load_text(root, "ROADMAP.md").split()) _require("`v0.3.0` is the" in roadmap, "ROADMAP release tag mismatch") _require( "current public specification release and is immutable" in roadmap, @@ -738,7 +725,7 @@ def validate_release_metadata(root: Path) -> None: ) _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") - frontend_03 = (root / FRONTEND_03_PROFILE_DOCUMENT).read_text(encoding="utf-8") + frontend_03 = " ".join(_load_text(root, FRONTEND_03_PROFILE_DOCUMENT).split()) _require( "Status: release candidate for the next OpenStatSpec minor release." in frontend_03 and "is not published stable until a protected or signed specification tag targets" in frontend_03 @@ -760,7 +747,9 @@ def validate_release_metadata(root: Path) -> None: f"{FRONTEND_03_PROFILE_DOCUMENT}: adapter claim gate is missing {phrase}", ) - classification = (root / "docs/spss-frontend-roadmap.md").read_text(encoding="utf-8") + classification = " ".join( + _load_text(root, "docs/spss-frontend-roadmap.md").split() + ) for phrase in ( "Frontend 0.3, Plan 0.1/0.2", "New Plan/Frontend/Binding generation", @@ -770,7 +759,10 @@ def validate_release_metadata(root: Path) -> None: "Milestone 1: Frontend 0.3 over Plan 0.1/0.2\n\n- [x] Specification-complete", "Adapter claims remain pending and separately gated", ): - _require(phrase in classification, f"SPSS classification is missing {phrase}") + _require( + " ".join(phrase.split()) in classification, + f"SPSS classification is missing {phrase}", + ) def validate_contract_artifacts(root: Path) -> ArtifactInventory: @@ -858,7 +850,7 @@ def validate_contract_artifacts(root: Path) -> ArtifactInventory: ) frontend_manifests[version] = manifest frontend_effective_cases = { - version: _expand_frontend_cases(root, version, manifest, frontend_manifests) + version: _expand_frontend_cases(version, manifest, frontend_manifests) for version, manifest in frontend_manifests.items() } frontend_plans = _resolve_frontend_plans( diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py index 3e314b1..eb5f2ce 100644 --- a/tests/test_contract_artifacts.py +++ b/tests/test_contract_artifacts.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import shutil @@ -8,8 +9,10 @@ import pytest from openstatspec_specification.artifacts import ArtifactValidationError +from openstatspec_specification.artifacts import canonical_json_bytes from openstatspec_specification.artifacts import validate_release_metadata from openstatspec_specification.artifacts import validate_contract_artifacts +from openstatspec_specification.artifacts import source_hash ROOT = Path(__file__).resolve().parents[1] @@ -65,6 +68,28 @@ def test_frontend_03_comment_and_varlist_case_inventory() -> None: assert inventory.frontend_effective_cases["0.3"] == 90 +def test_value_labels_then_add_value_labels_keeps_both_ordered_replacements() -> None: + manifest = json.loads( + (ROOT / "conformance/spss-syntax-frontend-0.3.json").read_text(encoding="utf-8") + ) + case = next( + case + for case in manifest["cases"] + if case["id"] == "value-labels-then-add-value-labels-uses-ordered-state" + ) + plan = case["expected_plan_0_1"] + assert [operation["op"] for operation in plan["operations"]] == [ + "replace_value_labels", + "replace_value_labels", + ] + assert plan["operations"][0]["labels"][0]["label"] == "Five" + assert plan["operations"][1]["labels"][-1]["label"] == "Two" + assert case["expected_source_hash"] == source_hash(case["request"]["source_text"]) + assert case["expected_plan_hash"] == hashlib.sha256( + canonical_json_bytes(plan) + ).hexdigest() + + def test_frontend_03_release_candidate_documentation_is_complete() -> None: readme = (ROOT / "README.md").read_text(encoding="utf-8") changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") @@ -120,6 +145,15 @@ def test_release_candidate_wording_fails_for_published_profile(tmp_path: Path) - validate_release_metadata(root) +def test_release_document_symlink_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + path.unlink() + path.symlink_to(root / "docs/transformation-plan-profile-0.2.md") + with pytest.raises(ArtifactValidationError, match="artifact path traverses symlink"): + validate_release_metadata(root) + + def copied_artifacts(tmp_path: Path) -> Path: for directory in ("conformance", "transformation", "sql", "docs"): shutil.copytree(ROOT / directory, tmp_path / directory) @@ -185,3 +219,53 @@ def test_binding_plan_and_frontend_references_must_agree(tmp_path: Path) -> None path.write_text(json.dumps(manifest), encoding="utf-8") with pytest.raises(ArtifactValidationError, match="binding reference mismatch"): validate_contract_artifacts(root) + + +def test_semantic_plan_failure_must_remain_schema_valid(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "reject-descending-range") + case["plan"]["operations"][0]["unexpected"] = True + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="semantic failure must have a schema-valid plan"): + validate_contract_artifacts(root) + + +def test_plan_schema_invalid_failure_requires_invalid_plan(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "reject-descending-range") + case["expected_error"] = "plan_schema_invalid" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="plan_schema_invalid requires a schema-invalid plan"): + validate_contract_artifacts(root) + + +def test_frontend_plan_schema_invalid_case_requires_valid_request_and_invalid_plan( + tmp_path: Path, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "formats-grouped-to-expands-source-order") + case["expected_error"] = "plan_schema_invalid" + case["expected_plan"]["operations"][0]["unexpected"] = True + case.pop("expected_plan_contract") + case.pop("expected_plan_hash") + path.write_text(json.dumps(manifest), encoding="utf-8") + validate_contract_artifacts(root) + + +def test_superseded_replacement_must_keep_request_identity(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + replacement = next( + case for case in manifest["cases"] if case["id"] == "inline-block-comment-is-ignored" + ) + replacement["request"]["input_alias"] = "other" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="superseded case request mismatch"): + validate_contract_artifacts(root)