diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43474af..d14f43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,4 +19,6 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 with: python-version: "3.13" + - run: python -m pip install --disable-pip-version-check -e ".[dev]" + - run: python -m pytest - run: python tools/validate_repository.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 65c0904..eabca06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,19 @@ ## Unreleased -Future changes will be listed here. +- SPSS Syntax Frontend 0.3 is a release candidate, not a published stable + claim. +- Added comments, existing-variable `TO`, grouped supported forms, predicate + aliases and bounded `NOT`, open recode ranges, and additive value labels. +- Frontend 0.3 emits only immutable Plan 0.1/0.2 objects and adds no SQL + binding operation. +- Repository validation now covers all Plan/Frontend/Binding schemas, + manifests, references, and hashes. ## v0.3.0 - 2026-08-12 -Release preparation for the planned stable `v0.3.0` tag. Until a protected or -signed `v0.3.0` tag targets this exact release commit, implementations MUST -identify it with `specification_status: release_candidate`, -`specification_release: null`, and the exact immutable specification commit. -The entries below become the stable release only when that tag exists. +OpenStatSpec `v0.3.0` is the stable public specification release at immutable +specification commit `cd8f198c68b849eb8ed018a894670a0904c2181d`. - Added backward-compatible Transformation Plan and SPSS Syntax Frontend 0.2 contracts for sequential numeric `COMPUTE` and conditional `IF`, bounded diff --git a/README.md b/README.md index c631ae9..c93888e 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,11 @@ permits a transformation to mutate a core import. That separate derived-data profile is not used by the SPSS-like in-place frontend below. -The release-candidate [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) -adds sequential bounded numeric assignment and conditional assignment to the +The released [Transformation Plan Profile 0.2](docs/transformation-plan-profile-0.2.md) +was released in OpenStatSpec `v0.3.0` and adds sequential bounded numeric assignment and conditional assignment to the unchanged 0.1 operations. The -[SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) +[released SPSS Syntax Frontend Profile 0.2](docs/spss-syntax-frontend-profile-0.2.md) +was released in OpenStatSpec `v0.3.0` and lowers `COMPUTE`, `IF`, `FORMATS`, `VARIABLE LEVEL`, and `EXECUTE` alongside the 0.1 `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS` subset. Programs using only the 0.1 subset retain exact 0.1 plan identity and hash. The @@ -50,6 +51,10 @@ creates no derived dataset, data copy, or OpenStatSpec undo layer; Dolt-specific history and commits remain Dolt's. MySQL, MariaDB, and Dolt require a new target to be provisioned separately before an in-place transformation apply. +The release-candidate [SPSS Syntax Frontend Profile 0.3](docs/spss-syntax-frontend-profile-0.3.md) +adds syntax-only expansion over the released Plan 0.1/0.2 contracts and is not +published stable yet. + ## Repository layout - `sql/server-version-policy.md` — server release-series claims and exact reference-CI targets. @@ -72,8 +77,18 @@ to be provisioned separately before an in-place transformation apply. - `transformation/plan-0.1.schema.json` — canonical transformation-plan schema. - `sql/transformation-plan-profile-schema.sql` — compact in-place apply audit; it is not a dataset-version catalog. -- `conformance/transformation-plan-0.2.json` and - `conformance/spss-syntax-frontend-0.2.json` — additive plan and frontend +- `docs/spss-syntax-frontend-profile-0.1.md`, + `docs/spss-syntax-frontend-profile-0.2.md`, and + `docs/spss-syntax-frontend-profile-0.3.md` — bounded syntax frontend + profiles and their contract boundaries. +- `transformation/spss-syntax-frontend-0.1.schema.json`, + `transformation/spss-syntax-frontend-0.2.schema.json`, and + `transformation/spss-syntax-frontend-0.3.schema.json` — frontend request + schemas. +- `conformance/transformation-plan-0.2.json`, + `conformance/spss-syntax-frontend-0.1.json`, + `conformance/spss-syntax-frontend-0.2.json`, and + `conformance/spss-syntax-frontend-0.3.json` — additive plan and frontend conformance cases with independent golden hashes; `conformance/in-place-transformation-0.2.json` fixes the same-dataset/same-table execution invariants and the additional controlled Dolt context. The 0.1 schemas and fixtures remain unchanged. diff --git a/ROADMAP.md b/ROADMAP.md index b862739..e8ebe49 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -65,6 +65,24 @@ Prerequisite: the SPSS profile, fixtures, and reference-adapter lessons are matu this does not block specification publication. - [ ] Evaluate additional language frontends as separate adapters that lower to the same canonical plan. +- [ ] Expand the bounded SPSS frontend through the independently versioned + milestones in the [SPSS frontend roadmap](docs/spss-frontend-roadmap.md), + without creating a general SPSS compatibility claim. +- [x] Complete the Frontend 0.3 specification candidate as a syntax-only + expansion over immutable Plan 0.1/0.2 after its schema, profile, 35 declared + cases, inherited compatibility, and repository validation are complete: + comments, dictionary-order variable lists, grouped supported forms, ordinary + not-equal/NOT predicates, open recode ranges, and additive value labels. +- [ ] Define typed numeric expressions, deterministic numeric/date functions, + missing-value predicates and metadata, additional numeric/date formats, and + restricted conditional blocks in a new Plan/Frontend/In-Place Binding + generation. +- [ ] Define string and physical-schema operations separately, with exact + add/rename/drop-column atomicity and rollback evidence for every claimed SQL + profile. +- [ ] Keep case-count/order and group/cross-dataset operations in separate + profiles whose invariants do not weaken the current same-cases in-place + binding. ## 8. Microsoft SQL Server dialect @@ -101,10 +119,8 @@ follows and does not block specification publication. record its `v0.3.0` release identifier; rebase conditional transformations on the lifecycle implementation and pass the combined service and conformance gates before making an adapter claim or package release. -5. [ ] Have PHP pin the exact specification commit and, once published, record - its `v0.3.0` release identifier; migrate to the canonical transformation plan - and pass the relevant conformance cases before making an adapter claim or - package release. +5. [ ] Have PHP pin the exact specification commit and pass applicable + conformance evidence before changing adapter or package-release claims. The pending in-place service matrix and adapter conformance work in sections 6 and 7 are downstream implementation gates. They do not block the `v0.3.0` diff --git a/conformance/spss-syntax-frontend-0.3.json b/conformance/spss-syntax-frontend-0.3.json new file mode 100644 index 0000000..2b2711e --- /dev/null +++ b/conformance/spss-syntax-frontend-0.3.json @@ -0,0 +1,714 @@ +{ + "manifest_version": "0.3", + "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.3", + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "plan_contracts": [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2" + ], + "request_schema": "../transformation/spss-syntax-frontend-0.3.schema.json", + "plan_schemas": { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json" + }, + "inherited_manifests": [ + { + "manifest": "spss-syntax-frontend-0.1.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" + } + }, + { + "manifest": "spss-syntax-frontend-0.2.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": {} + } + ], + "cases": [ + { + "id": "reject-comment-only-program", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* comment." + }, + "expected_source_hash": "8bfac9af8d0884b68cebc65a42360871592dd6e4e9cbead0bdf7029b6b64f4fc", + "expected_error": "spss_syntax_error" + }, + { + "id": "inline-block-comment-is-ignored", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* comment */." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "3c4ef459e9a2021100ef166ef0038c5f920b126d916db4c7273537d21b256f65", + "expected_error": null + }, + { + "id": "comment-command-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "COMMENT recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "5211e79a67d5f493b48a6907d365ade7b877d8351191c9924121af30f7865989", + "expected_error": null + }, + { + "id": "leading-star-comment-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "dddbaf0dd69f364ccc895d0d3b65daa63208edfd9b4e8e2112f915f1e73e5d70", + "expected_error": null + }, + { + "id": "comments-change-source-hash-not-plan-hash", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "COMMENT first comment.\n* second comment.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "8b0d8f677faa7e919c89ace749d0e45f89531fc6e22bfc796819c23081c1a5dc", + "expected_error": null + }, + { + "id": "reject-unterminated-inline-comment", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* unterminated." + }, + "expected_source_hash": "d675e34c7617e21d3b298bdc139d69fe2239fce0c82c03933da409c5a735f064", + "expected_error": "spss_syntax_error" + }, + { + "id": "recode-to-expands-dictionary-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "RECODE a TO c (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "recode", "source": "a", "target": "a", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}, + {"op": "recode", "source": "middle", "target": "middle", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}, + {"op": "recode", "source": "c", "target": "c", "target_mode": "replace", "rules": [{"match": {"kind": "values", "values": [{"type": "binary64", "bits": "3ff0000000000000"}]}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}} + ] + }, + "expected_plan_hash": "81c8b98d48b8c14ad43153bc52960e088c05a23fff3a6f60d01e3898650f2009", + "expected_source_hash": "365b8cadd07b148d7bbee906ce66ae608d3d3fd69ec82f6549623f724893b4f9", + "expected_error": null + }, + { + "id": "value-labels-to-expands-dictionary-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "VALUE LABELS a TO c 0 'No' 1 'Yes'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "replace_value_labels", "variable": "a", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"op": "replace_value_labels", "variable": "middle", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"op": "replace_value_labels", "variable": "c", "labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]} + ] + }, + "expected_plan_hash": "d3edd16ab980d21b1af06812fabf7f3e71d85972de9cb7f1717afac992450de1", + "expected_source_hash": "aa8a8028ac720e82794523bff70335c64e971cd8df7385ea2f6c9f60205bdbc0", + "expected_error": null + }, + { + "id": "formats-grouped-to-expands-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS a TO c (F8.2)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "set_format", "variable": "a", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "middle", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "c", "family": "F", "width": 8, "decimals": 2} + ] + }, + "expected_plan_hash": "c148754d88165fe10c5a1cd4c050a41b4e4a2839c8dad89531e972b69989aa7e", + "expected_source_hash": "9884908d306d12b46a02dd19001dba2ba3fc45b54a8ef40659cb586d0271d0d0", + "expected_error": null + }, + { + "id": "variable-level-to-expands-source-order", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "VARIABLE LEVEL a TO c (NOMINAL)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "set_measurement_level", "variable": "a", "level": "nominal"}, + {"op": "set_measurement_level", "variable": "middle", "level": "nominal"}, + {"op": "set_measurement_level", "variable": "c", "level": "nominal"} + ] + }, + "expected_plan_hash": "d3232768fb06358ae4f9e71da635908a133e19391fdaea125126e7d686ac1084", + "expected_source_hash": "1b1c7a9373fac15b8466e18a9f04c500e19747eb5e82a2ac3dc7b9301b450d09", + "expected_error": null + }, + { + "id": "to-sees-variable-created-by-preceding-command", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "COMPUTE appended = a.\nFORMATS c TO appended (F8.2)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [ + {"op": "assign", "target": "appended", "target_mode": "create", "value": {"kind": "variable", "variable": "a"}}, + {"op": "set_format", "variable": "c", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "outside", "family": "F", "width": 8, "decimals": 2}, + {"op": "set_format", "variable": "appended", "family": "F", "width": 8, "decimals": 2} + ] + }, + "expected_plan_hash": "97e447dee8786ba50661b8348ccfc2ec03d7d0f8d9f694daca0f3962dda6dd78", + "expected_source_hash": "819cd323cd9382e4340e4d4dcbbb47a5a155ccac674f2bfc616c7ca78e3c8cd5", + "expected_error": null + }, + { + "id": "reject-reversed-to-range", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS c TO a (F8.2)." + }, + "expected_source_hash": "b4a66bad42565222b32b559724ebefb029d427dfa49845d7e5ea1de05aad42f1", + "expected_error": "invalid_variable_range" + }, + { + "id": "reject-unknown-to-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "FORMATS a TO absent (F8.2)." + }, + "expected_source_hash": "6153086c283f905e177f45c9208e73e118315a6f8b496cfced05058da28009dd", + "expected_error": "unknown_variable" + }, + { + "id": "reject-to-generated-into-targets", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric"}, + {"name": "middle", "storage_kind": "numeric"}, + {"name": "c", "storage_kind": "numeric"}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "RECODE a (1 = 0) INTO target TO target2." + }, + "expected_source_hash": "c9b4ae4e9bfb7e25c8720f278d50669250fde123dffd4d743adaf18c91f83bd1", + "expected_error": "spss_syntax_error" + }, + { + "id": "ne-keyword-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a NE 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "e0292c5367e5b401224d4efd2a5acddaa05dbddb104cd37af5f0eb187f19c0aa", + "expected_error": null + }, + { + "id": "angle-not-equal-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a <> 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "343d43d13ed8b58623da9231054cf92005ea0048f1701bbd0dd2bf5822afc4c5", + "expected_error": null + }, + { + "id": "tilde-not-equal-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a ~= 1) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "9ba239c1692adb87db79b742e1f2aca74b002b1ed32466eb1366422c844d382b", + "expected_error": null + }, + { + "id": "not-equality-lowers-to-less-or-greater", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (source_a = 1)) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "93069b5cccb751d8e21d90ac02947a333b86c632149aebbdbe145b0588b41cf1", + "expected_source_hash": "e309a67df9b587b81766ed35afa770e99a0debc9918352e39f5d7cdb9058ba3e", + "expected_error": null + }, + { + "id": "not-nested-and-lowers-by-de-morgan", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "source_b", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (source_a = 1 AND source_b = 1)) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "boolean", "operator": "or", "operands": [ + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": "<", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, + {"expression": "comparison", "left": {"kind": "variable", "variable": "source_b"}, "operator": ">", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}} + ]}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "318eec6c5949cb7eb3b700727f00c0baee4cd521735b97443a207e42201082d6", + "expected_source_hash": "d6d61d8e9a27df8a0a48b8e88b8f7622cdb150a02f041449892d0a0b0c5311a4", + "expected_error": null + }, + { + "id": "double-not-retains-canonical-comparison", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT (NOT (source_a = 1))) target = 1.\nEXECUTE." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.2", + "expected_plan": { + "contract": "openstatspec-transformation-plan-v0.2", + "input_alias": "parent", + "operations": [{"op": "conditional_assign", "condition": {"expression": "comparison", "left": {"kind": "variable", "variable": "source_a"}, "operator": "=", "right": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, "target": "target", "value": {"kind": "literal", "value": {"type": "binary64", "bits": "3ff0000000000000"}}}, {"op": "execute"}] + }, + "expected_plan_hash": "fc0fc2836fcdec4852c869986182051c6ec39d44ca78621e3471b4c85330e122", + "expected_source_hash": "de80a1e13be7c1193afe63f88269a70ca50af471a5943b9126582053bb1a02a4", + "expected_error": null + }, + { + "id": "reject-not-numeric-truthiness", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (NOT source_a) target = 1." + }, + "expected_source_hash": "bf41ec574a901bc28462fc237edcdcb6bc1b88cc51af184d273c523951ca9518", + "expected_error": "expression_type_unsupported" + }, + { + "id": "reject-chained-comparison", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "source_a", "storage_kind": "numeric"}, + {"name": "source_b", "storage_kind": "numeric"}, + {"name": "source_c", "storage_kind": "numeric"}, + {"name": "target", "storage_kind": "numeric"} + ]}, + "source_text": "IF (source_a < source_b < source_c) target = 1." + }, + "expected_source_hash": "d5c8cab87b293826f708757132fcb629954ffdac6037cdd1f75f85b1400dab62", + "expected_error": "spss_syntax_error" + }, + { + "id": "recode-lowest-thru-upper", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (LOWEST THRU 1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "ffefffffffffffff"}, "upper": {"type": "binary64", "bits": "3ff0000000000000"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "e6b7fc9887b90f153067856771eaa3f8164a8e0212025f4c81923e70be6cec42", + "expected_source_hash": "7c9e0b869927dcda8a06e2b92f3c738902fabb93a126d2159583eeace55b3e3d", + "expected_error": null + }, + { + "id": "recode-lower-thru-highest", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (1 THRU HIGHEST = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "3ff0000000000000"}, "upper": {"type": "binary64", "bits": "7fefffffffffffff"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "ccfeaf45c8663888d2fd85a5c4808bbd8a314d1a2418554916a1f2746c0f2bea", + "expected_source_hash": "93b2464fb376be1c79ef49069b20252e40ec0d134b7b078148ee651da359acf1", + "expected_error": null + }, + { + "id": "recode-lowest-thru-highest", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (LOWEST THRU HIGHEST = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "recode", "source": "source_a", "target": "source_a", "target_mode": "replace", "rules": [{"match": {"kind": "range", "lower": {"type": "binary64", "bits": "ffefffffffffffff"}, "upper": {"type": "binary64", "bits": "7fefffffffffffff"}}, "result": {"kind": "literal", "value": {"type": "binary64", "bits": "0000000000000000"}}}], "unmatched": {"kind": "copy"}}] + }, + "expected_plan_hash": "6316b3fe33d13c034202c1472f1473d0658fb14538a1f5def8f83c9142c2adeb", + "expected_source_hash": "ac84fad5b5d0e29caef5e7e80ae83f5ee00404eca619a643840111d8571829a5", + "expected_error": null + }, + { + "id": "reject-string-open-range", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "string"}]}, + "source_text": "RECODE source_a (LOWEST THRU 1 = 'x')." + }, + "expected_source_hash": "cfd96fba97131220e5fef64551c8f4f746ff0019dd8e46174f3cfc301a4689ed", + "expected_error": "type_mismatch" + }, + { + "id": "reject-highest-as-lower-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (HIGHEST THRU 1 = 0)." + }, + "expected_source_hash": "44481c08e11c8ce299c3e1673e43047a438201cf7bcb2b0c9e3f38d99a00fbac", + "expected_error": "spss_syntax_error" + }, + { + "id": "reject-lowest-as-upper-endpoint", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "source_a", "storage_kind": "numeric"}]}, + "source_text": "RECODE source_a (1 THRU LOWEST = 0)." + }, + "expected_source_hash": "e9e9468f64c03bfd2822a365913399b3d5391f02f600e12606720b744a1d1e19", + "expected_error": "spss_syntax_error" + }, + { + "id": "add-value-labels-preserves-existing-and-appends", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}]}, + "source_text": "ADD VALUE LABELS q1 2 'Maybe'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}, + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"} + ]}] + }, + "expected_plan_hash": "520ec60bb84b6ec93eb4d2df1b7c0ba8a02efdcc9527284fabcd9b3ec26e931b", + "expected_source_hash": "c386c3ba15782601cf86459ec089d724ae7de449ee168b0d1acd06e489604b66", + "expected_error": null + }, + { + "id": "add-value-labels-replaces-existing-label-in-place", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}]}, + "source_text": "ADD VALUE LABELS q1 0 'Zero'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Zero"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"} + ]}] + }, + "expected_plan_hash": "0877ee65052235c31a57f76bafc4c647cee49bc75a43c89565a459ac86e33f7a", + "expected_source_hash": "b3899248d84f5229bb7ad410a748c209a5de6cce65dbc8c9c163e6cfc9d5a537", + "expected_error": null + }, + { + "id": "value-labels-then-add-value-labels-uses-ordered-state", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric", "value_labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "Original"} + ]}]}, + "source_text": "VALUE LABELS q1 5 'Five' 1 'One'.\nADD VALUE LABELS q1 2 'Two' 5 'Cinq'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Five"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"} + ]}, + {"op": "replace_value_labels", "variable": "q1", "labels": [ + {"value": {"type": "binary64", "bits": "4014000000000000"}, "label": "Cinq"}, + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "One"}, + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Two"} + ]} + ] + }, + "expected_plan_hash": "ed2c5ebaab637b6156f04f012bd99837e7377ac93e2919ec46e58fdc81cacd89", + "expected_source_hash": "fafcf4a687aa9241bf70b6152d6d10e083e357f322c7008c5aea8c72c5ccfd12", + "expected_error": null + }, + { + "id": "add-value-labels-multiple-variable-group", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [ + {"name": "a", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}]}, + {"name": "middle", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}]}, + {"name": "c", "storage_kind": "numeric", "value_labels": [{"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"}]}, + {"name": "outside", "storage_kind": "numeric"} + ]}, + "source_text": "ADD VALUE LABELS a TO c 3 'Other'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [ + {"op": "replace_value_labels", "variable": "a", "labels": [ + {"value": {"type": "binary64", "bits": "0000000000000000"}, "label": "No"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]}, + {"op": "replace_value_labels", "variable": "middle", "labels": [ + {"value": {"type": "binary64", "bits": "3ff0000000000000"}, "label": "Yes"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]}, + {"op": "replace_value_labels", "variable": "c", "labels": [ + {"value": {"type": "binary64", "bits": "4000000000000000"}, "label": "Maybe"}, + {"value": {"type": "binary64", "bits": "4008000000000000"}, "label": "Other"} + ]} + ] + }, + "expected_plan_hash": "7cc86bf992d568cde74802df7e6e68bc18f79210e084659e81b01d63f36ab2f5", + "expected_source_hash": "3353a38e495a448cdaad8bbb0981e4883bd21c9698148e0f87ad7158d13ca629", + "expected_error": null + }, + { + "id": "add-string-value-labels", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "status", "storage_kind": "string", "value_labels": [ + {"value": {"type": "string", "value": "N"}, "label": "No"} + ]}]}, + "source_text": "ADD VALUE LABELS status 'Y' 'Yes'." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_0_1": { + "contract": "openstatspec-transformation-plan-v0.1", + "input_alias": "parent", + "operations": [{"op": "replace_value_labels", "variable": "status", "labels": [ + {"value": {"type": "string", "value": "N"}, "label": "No"}, + {"value": {"type": "string", "value": "Y"}, "label": "Yes"} + ]}] + }, + "expected_plan_hash": "e53af9b03c09640d0b451879451eac5e25ccc7e8bf22836b8451650f6b790648", + "expected_source_hash": "a5cc8c9431e28c028f73874d985c7796733443baf1ac1832f533e39c81197b9e", + "expected_error": null + }, + { + "id": "reject-add-value-labels-duplicate-source-value", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "ADD VALUE LABELS q1 1 'One' 1 'Duplicate'." + }, + "expected_source_hash": "1b26095f77569a6d5e0ed785c777cd943850066e2075df642a24c834c9f49068", + "expected_error": "duplicate_value_label" + }, + { + "id": "reject-add-value-labels-type-mismatch", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "ADD VALUE LABELS q1 '1' 'One'." + }, + "expected_source_hash": "a0048e248ab54129fab1b6e3b528cf1ac220d1ab59a48a5547ae22b3eae45721", + "expected_error": "type_mismatch" + } + ] +} diff --git a/docs/spss-frontend-roadmap.md b/docs/spss-frontend-roadmap.md new file mode 100644 index 0000000..ec44993 --- /dev/null +++ b/docs/spss-frontend-roadmap.md @@ -0,0 +1,157 @@ +# SPSS Syntax Frontend Roadmap + +## Status and boundary + +This roadmap describes intended work, not a compatibility claim or release-date +promise. The current published boundary remains the exact bounded grammar in +[SPSS Syntax Frontend 0.2](spss-syntax-frontend-profile-0.2.md). Every syntax +form outside a claimed frontend contract must continue to fail closed. + +OpenStatSpec will expand the frontend in independently versioned stages. It will +not advertise general IBM SPSS Statistics compatibility, forward source +expressions into SQL, or weaken the same-dataset/same-table and caller-owned +Dolt history rules. + +Frontend 0.3 is specification-complete as a release candidate. Adapter claims +remain pending and separately gated; adapter evidence does not block +specification publication. + +## Command classification + +| Capability | Intended boundary | Reason | +| --- | --- | --- | +| Comments, existing-variable `TO`, grouped labels and recodes | Frontend 0.3, Plan 0.1 | Compile-time syntax expansion only | +| Grouped `FORMATS` and `VARIABLE LEVEL` | Frontend 0.3, Plan 0.2 | Expands to existing Plan 0.2 metadata operations | +| `NOT`, `NE`, `<>`, `~=` over the existing predicate grammar | Frontend 0.3, Plan 0.2 | Canonically lowers to existing comparisons and booleans | +| `LOWEST`/`HIGHEST` recode ranges | Frontend 0.3, Plan 0.1/0.2 | Lowers to finite binary64 endpoints; system missing remains distinct | +| `ADD VALUE LABELS` | Frontend 0.3, Plan 0.1/0.2 | Lowers ordered metadata state to complete label replacement | +| Arithmetic, deterministic numeric/date functions, missing predicates | New Plan/Frontend/Binding generation | Adds expression shapes and normative evaluation semantics | +| `MISSING VALUES` and additional numeric/date formats | New Plan/Frontend/Binding generation | Adds catalog operations and format semantics | +| Restricted `DO IF` | New Plan/Frontend/Binding generation | Requires branch-once and ordered-body semantics | +| `NUMERIC`, `STRING`, string expressions/functions, rename/delete | Separate Plan/Frontend/Binding generation | Adds types, physical DDL, catalog cleanup, and atomicity rules | +| `SELECT IF`, `SORT CASES` | Separate case-transformation profile | Changes current case-count/order invariants | +| Restricted add-variable aggregate and keyed lookup | Separate group/cross-dataset profile | Introduces group and multi-dataset semantics | +| `TEMPORARY`, `FILTER`, procedures, state/order-dependent functions | Explicit non-goal | Requires procedure/session state outside the mutation frontend | + +## Milestone 0: integrity and classification + +- [x] Specification-complete: Plan, Frontend, and In-Place schemas, manifests, + references, canonical bytes, and source/plan hashes are validated. +- [x] Add specification-owned validation for Plan, Frontend, and In-Place schemas, + manifests, references, canonical bytes, and source/plan hashes. +- Keep release status consistent across profile documents, roadmap, changelog, + tags, and releases. +- Classify every proposed command as frontend-only, new Plan/Binding work, a + separate profile, or an explicit non-goal before accepting syntax. + +Adapter claims remain pending and separately gated; adapter evidence does not +block specification publication. + +## Milestone 1: Frontend 0.3 over Plan 0.1/0.2 + +- [x] Specification-complete: the schema, profile, 35 declared cases, + inherited compatibility, and repository validation are present. +Frontend 0.3 adds syntax depth without adding plan operations: + +- `COMMENT`, leading-star comments, and inline block comments; +- existing-variable lists with dictionary-order `TO` expansion; +- grouped forms of supported labels and recodes (Plan 0.1), plus grouped `FORMATS` and `VARIABLE LEVEL` (Plan 0.2); +- `NOT`, `NE`, `<>`, and `~=` over the existing predicate grammar with + three-valued truth semantics; +- `LOWEST` and `HIGHEST` recode ranges; and +- `ADD VALUE LABELS`, lowered to a complete value-label replacement using the + ordered metadata state initialized from the request and updated by preceding + commands. + +The manifest must list only Plan 0.1 and Plan 0.2 outputs and prove that all +inherited programs retain their exact prior plans and hashes. No SQL binding or +executor change belongs in this milestone. + +Adapter claims remain pending and separately gated; adapter evidence does not +block specification publication. + +## Milestone 2: typed numeric expressions and metadata + +Define a new Plan, Frontend, and In-Place Binding generation for: + +- arithmetic parentheses, unary minus, `+`, `-`, `*`, and `/`; +- a deterministic numeric-function whitelist; +- `MISSING`, `SYSMIS`, `NMISS`, and `NVALID`; +- `MISSING VALUES` metadata; +- numeric `E`, `COMMA`, `DOT`, and `PCT` formats; and +- deterministic date construction, extraction, arithmetic, and display over + SPSS numeric serial values; and +- restricted `DO IF`/`ELSE IF`/`ELSE` blocks. + +The canonical expression model must define precedence, binary64 behavior, +domain errors, missing propagation, and sequential evaluation independently of +backend SQL behavior. Random, current-time, order-dependent, and +locale-dependent functions remain excluded. A conditional-block branch is +selected once from row state at block entry; its body then executes in source +order. + +## Milestone 3: strings and physical schema + +Define a separate contract milestone for: + +- `NUMERIC` and `STRING` declarations; +- string assignment, comparison, and a small deterministic function whitelist; +- simultaneous `RENAME VARIABLES`; and +- controlled `DELETE VARIABLES` with dependency checks and complete catalog + cleanup. + +The contract must specify fixed-width strings, blanks, padding, truncation, +Unicode indexing, collation, initialization, and physical/catalog identity. +Every SQL profile needs exact add/rename/drop-column atomicity and rollback +evidence. Non-atomic profiles retain explicit pre-provisioning for new targets. +Rename and delete fail before mutation where native DDL cannot satisfy the +contract; copied or staging tables are not recovery mechanisms. Dolt branch, +HEAD, clean-working-set, and caller-owned commit policy remain in force. + +## Milestone 4: case operations + +Treat row-count and row-order changes as a separate case-transformation profile: + +- `SELECT IF` may be considered as an explicit destructive operation with + affected-case evidence and exact UNKNOWN handling. +- `SORT CASES` waits for a persistent logical case-order model with stable + tie-breaking, missing ordering, and string collation. + +Neither command may be retrofitted into a binding that claims to preserve case +count and order. Physical SQL row order is not logical case order. + +## Milestone 5: group and cross-dataset operations + +Only bounded, row-count-preserving forms are candidates: + +- `AGGREGATE OUTFILE=* MODE=ADDVARIABLES` with a small deterministic function + set; and +- a unique-key lookup that adds selected columns to the active dataset without + changing its row count. + +Full `AGGREGATE OUTFILE`, unrestricted `MATCH FILES`, persistent outputs, +staging tables, snapshots, rollback copies, and hidden recovery datasets remain +outside this roadmap. + +## Explicit non-goals + +- Full SPSS syntax compatibility or statistical procedures. +- `TEMPORARY`, `FILTER`, weighting, `SPLIT FILE`, or procedure-scoped state. +- Random, current-time, `LAG`, cumulative, or environment-dependent functions. +- Macros, loops, vectors, extension-language execution, unrestricted joins, + reshaping, or `ALTER TYPE`. +- Any OpenStatSpec-managed persistent version or rollback layer. Dolt remains + the sole persistent version/history mechanism for successful edits. + +## Evidence sources + +The priorities are based on IBM's +[Command Syntax Reference](https://www.ibm.com/docs/en/SSLVMB_25.0.0/pdf/en/IBM_SPSS_Statistics_Command_Syntax_Reference.pdf), +[string-function documentation](https://www.ibm.com/docs/en/spss-statistics/25.0.0?topic=expressions-string-functions), +[FORMATS rules](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=formats-syntax-rules-command), +[SELECT IF reference](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-select-if), +[Aggregate Data documentation](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=transformations-aggregate-data), +and [MATCH FILES reference](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-match-files). +GNU PSPP and UCLA OARC documentation provide secondary implementation and +usage evidence; IBM semantics and OpenStatSpec's normative contracts remain +authoritative. diff --git a/docs/spss-syntax-frontend-profile-0.2.md b/docs/spss-syntax-frontend-profile-0.2.md index cab875c..2e05cfa 100644 --- a/docs/spss-syntax-frontend-profile-0.2.md +++ b/docs/spss-syntax-frontend-profile-0.2.md @@ -2,7 +2,8 @@ ## Status and boundary -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. This optional frontend maps a bounded SPSS-like command language to Transformation Plan 0.1 or 0.2. Its request contract is diff --git a/docs/spss-syntax-frontend-profile-0.3.md b/docs/spss-syntax-frontend-profile-0.3.md new file mode 100644 index 0000000..4cfa15e --- /dev/null +++ b/docs/spss-syntax-frontend-profile-0.3.md @@ -0,0 +1,164 @@ +# OpenStatSpec SPSS-like Syntax Frontend Profile 0.3 + +## Status and boundary + +Status: release candidate for the next OpenStatSpec minor release. This profile +is not published stable until a protected or signed specification tag targets +its exact reviewed commit and tag-context CI passes. + +This optional frontend profile maps a deliberately bounded SPSS-like command +language to Transformation Plan 0.1 or 0.2. Its request contract is +`openstatspec-spss-syntax-frontend-v0.3`. It is not full IBM SPSS Statistics +syntax and MUST reject every command, expression, or coercion not defined here. + +The request conforms to +[`../transformation/spss-syntax-frontend-0.3.schema.json`](../transformation/spss-syntax-frontend-0.3.schema.json). +All Frontend 0.2 lexical rules, command termination, exact source hashing, +case-insensitive name resolution, quote handling, and failure atomicity remain +in force. The input schema retains only ordered variables, storage kind, +variable and value labels, format family, width, decimals, and measurement +level. It does not declare physical identifiers. + +## Compatibility and output-plan selection + +Frontend 0.3 consumes unchanged Frontend 0.2 input-schema fields and unchanged +Plan 0.1 and Plan 0.2 contracts. Comments, `TO`, grouped recodes and labels, +open-ended RECODE ranges, and additive value labels remain in the Plan 0.1 +output subset. Grouped `FORMATS` and `VARIABLE LEVEL` emit Plan 0.2 +`set_format` and `set_measurement_level` operations, respectively. Predicate +aliases and `NOT` require the existing Plan 0.2 `conditional_assign` syntax. +A program containing only commands and productions whose underlying operations +are in the Plan 0.1 subset MUST emit an exact Plan 0.1 object. A program +containing any Plan 0.2-only operation MUST emit Plan 0.2. No command is +dropped, and the frontend performs no data mutation, SQL execution, branch +operation, or Dolt commit. + +## Comments and source identity + +Leading-star comments are recognized only when `*` is the first non-whitespace +token at a command boundary. Such a comment ends at the next command period. +The `COMMENT` command is a command comment and ends at its command period. +Non-nested `/* ... */` comments are accepted wherever whitespace is legal. +Unterminated or nested block comments MUST return `spss_syntax_error`. + +Comments are retained in the LF-normalized source used for source hashing and +emit no operation. A comment-only program MUST return `spss_syntax_error` +because a valid plan cannot have zero operations. Source hashing remains +SHA-256 of UTF-8 source after CRLF and CR are converted to LF. The complete +emitted plan is hashed independently under its declared plan contract. + +## Existing-variable lists + +The bounded grammar is: + +```text +existing-variable-list := existing-variable + ((existing-variable | TO existing-variable))* +comparison-operator := "=" | "<" | "<=" | ">" | ">=" + | "NE" | "<>" | "~=" +predicate := comparison | "(" predicate ")" + | NOT predicate + | predicate AND predicate + | predicate OR predicate +open-range := LOWEST THRU finite-number + | finite-number THRU HIGHEST + | LOWEST THRU HIGHEST +``` + +`TO` expands inclusively using the current ordered input-schema state, not +variable spelling or a numeric suffix. Reverse ranges MUST return +`invalid_variable_range`; absent endpoints MUST return `unknown_variable`. +`TO` is forbidden for generated `INTO` target sequences. Resolved variables +retain their catalog spelling and source order; each occurrence is resolved at +the point its command is lowered. + +## Predicate aliases and NOT lowering + +`NE`, `<>`, and `~=` are aliases for ordered `< OR >` comparisons. `NOT` is +accepted only over the existing comparison and boolean grammar. It lowers +recursively using comparison complements and De Morgan's laws, while +preserving SQL three-valued truth. The complements are `=` to `< OR >`, `<` to +`>=`, `<=` to `>`, `>` to `<=`, and `>=` to `<`; `NE`, `<>`, and `~=` first +lower to `< OR >`. Boolean negation swaps `AND` and `OR` recursively. + +The canonical lowering table is: + +```text +NOT (left = right) -> (left < right OR left > right) +NOT (left < right) -> left >= right +NOT (left <= right) -> left > right +NOT (left > right) -> left <= right +NOT (left >= right) -> left < right +NOT (a AND b) -> NOT a OR NOT b +NOT (a OR b) -> NOT a AND NOT b +``` + +Canonical lowering flattens maximal same-operator boolean expressions while +preserving operand order. Parentheses override precedence across different +operators, but parentheses around a same-operator expression do not prevent +required flattening. Missing comparisons remain UNKNOWN and no lowering may +turn UNKNOWN into TRUE or FALSE. + +## Open-ended RECODE ranges + +`RECODE` retains the Frontend 0.2 typed selector and result rules. In addition, +numeric selectors may use the open-range forms in the grammar above. `LOWEST` +is the finite binary64 value with bits `ffefffffffffffff`, and `HIGHEST` is the +finite binary64 value with bits `7fefffffffffffff`. These bounds include every +finite binary64 value in the corresponding direction. SQL NULL/system missing +is not included in an open range. + +Open ranges are inclusive and lower to the existing ordered `recode` range +representation. Non-finite values, malformed finite numbers, and reverse +ranges are rejected with `spss_syntax_error` or `invalid_variable_range` as +applicable. The finite-number token remains the exact ASCII grammar inherited +from Frontend 0.2 and is interpreted as an exact base-ten value before +ties-to-even binary64 conversion. + +## ADD VALUE LABELS + +`ADD VALUE LABELS variable... value 'label' [value 'label' ...] +[/ variable... value 'label' ...].` resolves each existing variable against +the current typed schema. An existing typed value is updated at its current +ordinal. A new typed value is appended in source order. Duplicate values in a +single source group are invalid. Typed values use exact canonical identity, +including positive-zero numeric canonicalization and exact string contents. + +The frontend emits one complete `replace_value_labels` operation per variable, +in source variable order, preserving all existing labels and applying the +updates and appended values. Variable and value types MUST agree; no implicit +numeric/string coercion is performed. If a preceding `VALUE LABELS` command +replaced the label state, a following `ADD VALUE LABELS` command merges against +that replacement state. The command remains in the Plan 0.1 output subset. + +## Failure contract + +The frontend MUST produce no partial plan after any error. In addition to the +inherited 0.1 and 0.2 diagnostics, stable diagnostics include +`invalid_variable_range`, `unknown_variable`, and `spss_syntax_error` for the +rules defined here. Unsupported commands or productions return +`unsupported_spss_command` or `spss_syntax_error` as appropriate. Diagnostics +identify source spans but MUST NOT include credentials or unrelated row values. + +## Conformance + +Conformance cases for this profile are published in +[`../conformance/spss-syntax-frontend-0.3.json`](../conformance/spss-syntax-frontend-0.3.json). +The manifest declares 35 Frontend 0.3 cases and expands to 90 effective cases +after inherited cases and the two published comment supersessions are applied. + +## Adapter claim gate + +An adapter claim for this profile MUST satisfy every gate below: + +1. Pin the exact future specification release commit. +2. Accept only the exact Frontend 0.3 request contract for this suite. +3. Run all 90 effective Frontend 0.3 cases. +4. Preserve every inherited Plan 0.1/0.2 object and hash. +5. Pass all 35 declared cases with exact source/plan hashes and diagnostics. +6. Continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites. +7. Keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy + unchanged. + +Adapter evidence does not block specification publication. The specification +release-candidate gate and each downstream adapter claim gate are separate. diff --git a/docs/superpowers/plans/2026-08-21-spss-frontend-03.md b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md new file mode 100644 index 0000000..106c049 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-spss-frontend-03.md @@ -0,0 +1,1361 @@ +# SPSS Frontend 0.3 Specification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Publish a complete specification candidate for SPSS Syntax Frontend 0.3 that adds bounded syntax depth while emitting only immutable Transformation Plan 0.1/0.2 objects. + +**Architecture:** First add repository-owned validation for every Plan, Frontend, and In-Place artifact and repair release-status bookkeeping. Then add an exact Frontend 0.3 request schema, normative profile, inherited-case manifest, and independently authored golden cases. Frontend 0.3 changes accepted source syntax only; it adds no Plan operation, SQL binding, audit schema, or executor behavior. + +**Tech Stack:** Python 3.11+, pytest 8, jsonschema Draft 2020-12, JSON Schema, restricted RFC 8785 canonical JSON, SHA-256, Markdown, GitHub Actions. + +**Spec:** `docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md` + +## Global Constraints + +- Published Plan 0.1/0.2 schemas, operations, canonical bytes, hashes, and diagnostics are immutable. +- Published Frontend 0.1/0.2 request schemas and manifest cases are immutable. +- Contract identifiers select exact behavior; Frontend 0.3 uses `openstatspec-spss-syntax-frontend-v0.3`. +- Frontend 0.3 emits only `openstatspec-transformation-plan-v0.1` or `openstatspec-transformation-plan-v0.2`. +- A program inside the Plan 0.1 operation subset emits the exact Plan 0.1 object, even when its request contract is Frontend 0.3. +- Comments remain material to `source_hash` but emit no operation. +- Every undefined command, expression, coercion, comment form, or variable-list form fails closed. +- No task adds arithmetic, functions, missing predicates, string expressions, schema operations, case deletion/order, aggregation, joins, persistent copies, or rollback state. +- Specification publication precedes adapter claims; PHP and Python implementation work is planned separately after this contract is reviewed and published. +- Dolt remains caller-owned history; this frontend-only release does not change branch, HEAD, working-set, commit, or pre-provisioning rules. + +## File Structure + +- `src/openstatspec_specification/artifacts.py`: repository artifact loader, JSON Schema validation, canonical hashing, manifest reference validation, inherited-case expansion, and inventory reporting. +- `tests/test_contract_artifacts.py`: positive inventory checks and mutation-based fail-closed tests for schemas, hashes, IDs, references, inheritance, and release status. +- `tools/validate_repository.py`: invokes the artifact validator in addition to existing Dolt and repository controls. +- `pyproject.toml`: declares `jsonschema` and the pytest development dependency. +- `.github/workflows/ci.yml`: installs the package and runs pytest before repository validation. +- `transformation/spss-syntax-frontend-0.3.schema.json`: exact Frontend 0.3 request envelope; structurally identical to 0.2 except contract identity and schema metadata. +- `docs/spss-syntax-frontend-profile-0.3.md`: normative lexical, grammar, lowering, diagnostics, compatibility, and hashing rules. +- `conformance/spss-syntax-frontend-0.3.json`: inherited 0.1/0.2 cases, explicit supersession records, and 35 new independent golden cases. +- `examples/spss-syntax-transformation-0.3.md`: one readable program showing comments, `TO`, grouped format, `NOT`/not-equal, open recode ranges, and additive labels. +- `README.md`, `CHANGELOG.md`, `ROADMAP.md`, `docs/spss-frontend-roadmap.md`: artifact index, release notes, milestone status, and adapter handoff gates. +- `docs/transformation-plan-profile-0.2.md`, `docs/spss-syntax-frontend-profile-0.2.md`, `docs/transformation-plan-sql-binding-0.2.md`: correct published `v0.3.0` status wording only; no normative edits. + +--- + +### Task 1: Add the contract artifact validation foundation + +**Files:** +- Create: `src/openstatspec_specification/artifacts.py` +- Create: `tests/test_contract_artifacts.py` +- Modify: `pyproject.toml` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- Produces: `ArtifactValidationError`, `ArtifactInventory`, `canonical_json_bytes(value: object) -> bytes`, `source_hash(source_text: str) -> str`, and `validate_contract_artifacts(root: Path) -> ArtifactInventory`. +- Consumes: existing files below `transformation/`, `conformance/`, `sql/`, and `docs/`; no adapter code. + +- [ ] **Step 1: Add a failing inventory test.** + +Create `tests/test_contract_artifacts.py` with the existing released inventory as the first executable requirement: + +```python +from __future__ import annotations + +from pathlib import Path + +from openstatspec_specification.artifacts import validate_contract_artifacts + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_released_contract_artifact_inventory_is_valid() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.plan_cases == {"0.1": 4, "0.2": 26} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44} + assert inventory.binding_cases == {"0.1": 6, "0.2": 11} +``` + +- [ ] **Step 2: Run the focused test and confirm the missing-module failure.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: collection fails with `ModuleNotFoundError: No module named 'openstatspec_specification.artifacts'`. + +- [ ] **Step 3: Declare exact validation dependencies and CI setup.** + +Change `pyproject.toml` to include: + +```toml +dependencies = ["jsonschema>=4.23,<5"] + +[project.optional-dependencies] +dev = ["pytest>=8,<9"] +``` + +In `.github/workflows/ci.yml`, insert these commands after `actions/setup-python` and before repository validation: + +```yaml + - run: python -m pip install --disable-pip-version-check -e ".[dev]" + - run: python -m pytest + - run: python tools/validate_repository.py +``` + +Remove the old duplicate standalone `python tools/validate_repository.py` step. + +- [ ] **Step 4: Implement the loader, schema checks, top-level contracts, and unique IDs.** + +Create `src/openstatspec_specification/artifacts.py` with these public types and constants: + +```python +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + + +PLAN_MANIFESTS = { + "0.1": "conformance/transformation-plan-0.1.json", + "0.2": "conformance/transformation-plan-0.2.json", +} +FRONTEND_MANIFESTS = { + "0.1": "conformance/spss-syntax-frontend-0.1.json", + "0.2": "conformance/spss-syntax-frontend-0.2.json", +} +BINDING_MANIFESTS = { + "0.1": "conformance/in-place-transformation-0.1.json", + "0.2": "conformance/in-place-transformation-0.2.json", +} + + +class ArtifactValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class ArtifactInventory: + plan_cases: Mapping[str, int] + frontend_declared_cases: Mapping[str, int] + frontend_effective_cases: Mapping[str, int] + binding_cases: Mapping[str, int] + + +def canonical_json_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def source_hash(source_text: str) -> str: + normalized = source_text.replace("\r\n", "\n").replace("\r", "\n") + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() +``` + +Implement private helpers with these exact responsibilities: + +```python +def _load_json(root: Path, relative: str) -> object: + parts = relative.split("/") + _require( + not relative.startswith("/") + and "\\" not in relative + and all(part not in {"", ".", ".."} for part in parts), + f"noncanonical artifact path: {relative}", + ) + candidate = root.joinpath(*parts) + current = root + for part in parts: + current = current / part + _require(not current.is_symlink(), f"artifact path traverses symlink: {relative}") + _require(candidate.is_file(), f"artifact is missing: {relative}") + try: + return json.loads(candidate.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactValidationError(message) + +def _resolve_manifest_reference(manifest_relative: str, reference: str) -> str: + _require( + isinstance(reference, str) + and bool(reference) + and not reference.startswith("/") + and "\\" not in reference, + f"noncanonical manifest reference: {reference!r}", + ) + resolved: list[str] = [] + candidate = PurePosixPath(manifest_relative).parent / reference + for part in candidate.parts: + if part in {"", "."}: + continue + if part == "..": + _require(bool(resolved), f"manifest reference escapes repository root: {reference}") + resolved.pop() + continue + resolved.append(part) + _require(bool(resolved), f"manifest reference resolves to repository root: {reference}") + return "/".join(resolved) + +def _require_unique_case_ids(manifest: Mapping[str, object], context: str) -> None: + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{context}: cases must be a list") + seen: set[str] = set() + for case in cases: + _require(isinstance(case, dict), f"{context}: each case must be an object") + case_id = case.get("id") + _require(isinstance(case_id, str) and bool(case_id), f"{context}: case id must be a non-empty string") + _require(case_id not in seen, f"{context}: duplicate case id: {case_id}") + seen.add(case_id) + +def _schema(root: Path, relative: str) -> Mapping[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: schema must be an object") + try: + Draft202012Validator.check_schema(value) + except SchemaError as error: + raise ArtifactValidationError(f"{relative}: invalid JSON Schema: {error.message}") from None + return value +``` + +`validate_contract_artifacts()` must load all six existing manifests, require their exact `manifest_version`, `contract`, and relative schema fields, meta-validate all four referenced Plan/Frontend schemas, require unique case IDs, and return the released counts shown in Step 1. Convert `OSError`, `UnicodeError`, `JSONDecodeError`, `SchemaError`, and `ValidationError` into `ArtifactValidationError` without leaking unrelated filesystem data. + +Resolve every manifest-owned schema or audit reference through +`_resolve_manifest_reference(manifest_relative, reference)` before passing the +result to `_load_json()`. Direct repository-relative loader calls continue to +reject `..`; only the resolver may consume and normalize manifest-relative +parent segments, and it must never permit escape above repository root. + +- [ ] **Step 5: Add fail-closed tests for malformed schemas and duplicate IDs.** + +Add this copy helper and tests: + +```python +import json +import shutil + +import pytest + +from openstatspec_specification.artifacts import ArtifactValidationError + + +def copied_artifacts(tmp_path: Path) -> Path: + for directory in ("conformance", "transformation", "sql", "docs"): + shutil.copytree(ROOT / directory, tmp_path / directory) + shutil.copy2(ROOT / "CHANGELOG.md", tmp_path / "CHANGELOG.md") + shutil.copy2(ROOT / "ROADMAP.md", tmp_path / "ROADMAP.md") + return tmp_path + + +def test_duplicate_manifest_case_id_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"].append(manifest["cases"][0]) + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="duplicate case id"): + validate_contract_artifacts(root) + + +def test_invalid_json_schema_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "transformation/plan-0.1.schema.json" + schema = json.loads(path.read_text(encoding="utf-8")) + schema["type"] = "not-a-json-schema-type" + path.write_text(json.dumps(schema), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="invalid JSON Schema"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 6: Run the focused and full tests.** + +Run: + +```bash +python3 -m pip install --disable-pip-version-check -e ".[dev]" +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +``` + +Expected: all tests pass; inventory remains Plan `4/26`, Frontend `13/44`, Binding `6/11`. + +- [ ] **Step 7: Commit the validation foundation.** + +```bash +git add pyproject.toml .github/workflows/ci.yml src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "test: validate contract artifact inventory" +``` + +### Task 2: Validate canonical hashes and cross-manifest references + +**Files:** +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `tools/validate_repository.py` + +**Interfaces:** +- Consumes: Task 1 `validate_contract_artifacts()`, `canonical_json_bytes()`, and `source_hash()`. +- Produces: exact validation of plan hashes, frontend source hashes, referenced plans, binding references, canonical audit JSON, and failure-case shape. + +- [ ] **Step 1: Add failing mutation tests for plan hash, source hash, and references.** + +```python +@pytest.mark.parametrize( + ("manifest_name", "field"), + [ + ("transformation-plan-0.2.json", "expected_plan_hash"), + ("spss-syntax-frontend-0.2.json", "expected_source_hash"), + ], +) +def test_hash_mismatch_fails_closed( + tmp_path: Path, manifest_name: str, field: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance" / manifest_name + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0][field] = "0" * 64 + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="hash mismatch"): + validate_contract_artifacts(root) + + +def test_missing_frontend_plan_reference_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["expected_plan_case"] = "absent-plan" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="unknown plan case"): + validate_contract_artifacts(root) + + +def test_binding_plan_and_frontend_references_must_agree(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/in-place-transformation-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["applied_plan_case"] = "strict-greater-than" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="binding reference mismatch"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 2: Run the focused tests and confirm all three mutations are accepted incorrectly.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: the three new tests fail because Task 1 does not inspect hashes or references. + +- [ ] **Step 3: Implement Plan manifest validation.** + +For every Plan manifest: + +1. Index cases by ID. +2. Require `expected_error` to be either `null` or a non-empty string. +3. For successful cases, require a `plan` object, validate it against the manifest's schema, calculate `sha256(canonical_json_bytes(plan))`, and require exact lowercase `expected_plan_hash` equality. +4. For `plan_schema_invalid` cases, require a plan object that fails the selected Plan schema, with `expected_plan_hash` absent or `null`. +5. For semantic failures, require a plan object that passes the selected Plan schema, retain its non-empty `expected_error`, and omit only the success hash. +6. Require the plan's `contract` to match the manifest contract for successful plan objects; a schema-invalid negative may fail this contract constraint as part of its selected-schema failure. + +Add private helpers with these signatures: + +```python +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + +def _validate_plan_manifests( + root: Path, + manifests: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, object]]: + """Return version -> case-id -> successful canonical plan case.""" +``` + +- [ ] **Step 4: Implement Frontend request, source hash, and plan-reference validation.** + +For each Frontend case: + +1. Compute `source_hash(request["source_text"])` for every structurally readable request and require exact `expected_source_hash`. +2. Validate successful requests against `request_schema`. +3. Require every readable request to pass its request schema, including `plan_schema_invalid` cases; malformed request cases remain invalid requests and do not become plan-schema cases. +4. Resolve a successful or `plan_schema_invalid` expected plan from exactly one of `expected_plan`, `expected_plan_0_1`, `expected_plan_case`, or `expected_plan_case_0_1`. +5. Resolve `expected_plan_case` against the declared Plan manifest and `expected_plan_case_0_1` against Frontend 0.1's resolved expected plans. +6. Validate the resolved plan against the schema selected by its exact `contract`, recompute its canonical hash, and require equality with `expected_plan_hash`. +7. Require semantic frontend failures to omit expected plan objects, references, and hashes. A Frontend `plan_schema_invalid` case instead includes the invalid expected plan, selects a recognized Plan contract, validates the request structurally, and omits or nulls `expected_plan_hash`. + +Use this exact resolver signature: + +```python +def _resolve_frontend_plans( + root: Path, + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], + effective_cases: Mapping[str, tuple[Mapping[str, object], ...]], +) -> Mapping[str, Mapping[str, Mapping[str, object]]]: + """Return version -> case-id -> resolved canonical plan for successes.""" +``` + +The fourth argument is the version-to-effective-case mapping produced by +`_expand_frontend_cases`; it includes inherited cases with namespaced IDs and +declared replacements, after superseded cases are removed. The resolver returns +only structurally valid successful plans. Semantic negative frontend cases keep +their `expected_error` and no plan fields. A `plan_schema_invalid` case has a +schema-valid request and one inline invalid plan field, with a recognized plan +contract and no expected plan hash. + +- [ ] **Step 5: Implement In-Place reference and audit identity validation.** + +For In-Place 0.2 cases containing `applied_plan_case` and +`applied_frontend_case`, require both references to exist, require the frontend +case to resolve to that exact plan case, and verify these audit fields when +present: + +```python +expected_audit["plan_hash"] == plan_case["expected_plan_hash"] +expected_audit["source_hash"] == frontend_case["expected_source_hash"] +expected_audit["canonical_plan_json"].encode("utf-8") == canonical_json_bytes(plan_case["plan"]) +expected_audit["operation_count"] == len(plan_case["plan"]["operations"]) +``` + +Require every In-Place manifest case to have a unique ID and exact manifest +contract. Require every failing binding case to contain a non-empty +`expected_error` and `mutation_started: false`. + +- [ ] **Step 6: Wire artifact validation into the repository command.** + +In `tools/validate_repository.py`, import and call the validator after the +existing reusable Dolt validator and before phrase controls: + +```python +from openstatspec_specification.artifacts import ( # noqa: E402 + ArtifactValidationError, + validate_contract_artifacts, +) + + +if __name__ == "__main__": + try: + main() + inventory = validate_contract_artifacts(ROOT) + validate_repository_controls() + print( + "Validated contract artifacts: " + f"Plan {dict(inventory.plan_cases)}, " + f"Frontend {dict(inventory.frontend_effective_cases)}, " + f"Binding {dict(inventory.binding_cases)}." + ) + except (DoltDeclarationError, ArtifactValidationError) as error: + raise SystemExit(str(error)) from error +``` + +- [ ] **Step 7: Run mutation tests and the complete repository gate.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: all tests pass; the repository command prints all six released inventory counts and exits zero. + +- [ ] **Step 8: Commit canonical artifact validation.** + +```bash +git add src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py tools/validate_repository.py +git commit -m "test: verify transformation artifact identity" +``` + +### Task 3: Repair release status and lock command classification + +**Files:** +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/transformation-plan-profile-0.2.md` +- Modify: `docs/spss-syntax-frontend-profile-0.2.md` +- Modify: `docs/transformation-plan-sql-binding-0.2.md` +- Modify: `CHANGELOG.md` +- Modify: `ROADMAP.md` +- Verify: `docs/spss-frontend-roadmap.md` + +**Interfaces:** +- Consumes: published `v0.3.0` tag target `cd8f198c68b849eb8ed018a894670a0904c2181d` and the command-classification table already in `docs/spss-frontend-roadmap.md`. +- Produces: `validate_release_metadata(root: Path) -> None` and consistent released status for all 0.2 profiles. + +- [ ] **Step 1: Add a failing release-status consistency test.** + +```python +from openstatspec_specification.artifacts import validate_release_metadata + + +def test_v030_profile_status_is_consistently_released() -> None: + validate_release_metadata(ROOT) + + +def test_release_candidate_wording_fails_for_published_profile(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + text = path.read_text(encoding="utf-8").replace( + "Status: released in OpenStatSpec `v0.3.0`", + "Status: release candidate for OpenStatSpec `v0.3.0`", + ) + path.write_text(text, encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="release status"): + validate_release_metadata(root) +``` + +- [ ] **Step 2: Run the test and confirm current release-candidate wording fails.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: `test_v030_profile_status_is_consistently_released` fails because three published 0.2 documents still say release candidate. + +- [ ] **Step 3: Implement exact release metadata validation.** + +Add: + +```python +V030_COMMIT = "cd8f198c68b849eb8ed018a894670a0904c2181d" +V030_PROFILE_DOCUMENTS = ( + "docs/transformation-plan-profile-0.2.md", + "docs/spss-syntax-frontend-profile-0.2.md", + "docs/transformation-plan-sql-binding-0.2.md", +) + + +def validate_release_metadata(root: Path) -> None: + for relative in V030_PROFILE_DOCUMENTS: + text = (root / relative).read_text(encoding="utf-8") + _require( + "Status: released in OpenStatSpec `v0.3.0`" in text, + f"{relative}: release status is inconsistent with v0.3.0", + ) + _require( + "release candidate for the planned OpenStatSpec `v0.3.0`" not in text, + f"{relative}: stale release status remains", + ) + roadmap = (root / "ROADMAP.md").read_text(encoding="utf-8") + _require("`v0.3.0` is the" in roadmap, "ROADMAP release tag mismatch") + _require("current public specification release and is immutable" in roadmap, "ROADMAP release status mismatch") + _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") +``` + +Extend the `tools/validate_repository.py` artifact import with +`validate_release_metadata`, then call `validate_release_metadata(ROOT)` from +`validate_repository_controls()`. + +- [ ] **Step 4: Correct only release bookkeeping text.** + +Replace each 0.2 profile's status paragraph with: + +```markdown +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. +``` + +In `CHANGELOG.md`, replace the conditional release-preparation paragraph under +`v0.3.0` with a factual released statement naming the same tag and commit. Do +not edit any normative command, plan, binding, or diagnostic text. + +- [ ] **Step 5: Record completed PHP downstream evidence without changing specification gates.** + +In `ROADMAP.md` section 9, mark the PHP item complete and cite: + +```markdown +5. [x] PHP pinned specification `v0.3.0` commit + `cd8f198c68b849eb8ed018a894670a0904c2181d`, passed its complete PHP + 8.4/8.5 and PostgreSQL/MySQL/MariaDB/Dolt matrix, and published adapter + [v0.6.0](https://github.com/OpenStatSpec/php/releases/tag/v0.6.0). +``` + +Leave Python status unchanged unless its own exact release evidence is independently verified in that task. + +- [ ] **Step 6: Verify the public command-classification table is complete.** + +Require `docs/spss-frontend-roadmap.md` to contain all four boundary phrases: + +```python +classification = (root / "docs/spss-frontend-roadmap.md").read_text(encoding="utf-8") +for phrase in ( + "Frontend 0.3, Plan 0.1/0.2", + "New Plan/Frontend/Binding generation", + "Separate case-transformation profile", + "Explicit non-goal", +): + _require(phrase in classification, f"SPSS classification is missing {phrase}") +``` + +- [ ] **Step 7: Run all gates and commit bookkeeping.** + +Run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +git diff --check +``` + +Then commit: + +```bash +git add src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py docs/transformation-plan-profile-0.2.md docs/spss-syntax-frontend-profile-0.2.md docs/transformation-plan-sql-binding-0.2.md CHANGELOG.md ROADMAP.md +git commit -m "docs: align v0.3.0 release status" +``` + +### Task 4: Define the Frontend 0.3 request contract and normative grammar + +**Files:** +- Create: `transformation/spss-syntax-frontend-0.3.schema.json` +- Create: `docs/spss-syntax-frontend-profile-0.3.md` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Produces: request contract `openstatspec-spss-syntax-frontend-v0.3` and its exact Draft 2020-12 schema. +- Consumes: unchanged Frontend 0.2 input-schema fields and unchanged Plan 0.1/0.2 contracts. + +- [ ] **Step 1: Add failing schema inventory tests.** + +```python +def test_frontend_03_request_schema_is_registered() -> None: + inventory = validate_contract_artifacts(ROOT) + assert "0.3" in inventory.frontend_declared_cases + + +def test_frontend_03_schema_changes_only_contract_identity() -> None: + old = json.loads((ROOT / "transformation/spss-syntax-frontend-0.2.schema.json").read_text()) + new = json.loads((ROOT / "transformation/spss-syntax-frontend-0.3.schema.json").read_text()) + assert new["$id"].endswith("spss-syntax-frontend-0.3.schema.json") + assert new["title"] == "OpenStatSpec SPSS-like Syntax Frontend Request 0.3" + assert new["properties"]["contract"]["const"] == "openstatspec-spss-syntax-frontend-v0.3" + for document in (old, new): + document.pop("$id") + document.pop("title") + document["properties"]["contract"].pop("const") + assert new == old +``` + +- [ ] **Step 2: Run the test and confirm the 0.3 schema/manifest is absent.** + +Run: `python3 -m pytest tests/test_contract_artifacts.py -q` + +Expected: failure because no Frontend 0.3 schema or registered manifest exists. + +- [ ] **Step 3: Create the exact Frontend 0.3 request schema.** + +Copy Frontend 0.2 structurally and change only: + +```json +{ + "$id": "https://openstatspec.org/schemas/spss-syntax-frontend-0.3.schema.json", + "title": "OpenStatSpec SPSS-like Syntax Frontend Request 0.3", + "properties": { + "contract": {"const": "openstatspec-spss-syntax-frontend-v0.3"} + } +} +``` + +Retain all 0.2 input fields and `$defs`, including ordered variables, storage +kind, variable/value labels, format family/width/decimals, and measurement +level. Do not add missing rules or physical identifiers in 0.3. + +- [ ] **Step 4: Write the normative 0.3 profile with exact grammar boundaries.** + +Create `docs/spss-syntax-frontend-profile-0.3.md` with these sections and rules: + +```text +Status and boundary +Compatibility and output-plan selection +Comments and source identity +Existing-variable lists +Predicate aliases and NOT lowering +Open-ended RECODE ranges +ADD VALUE LABELS +Failure contract +Conformance +``` + +Include this bounded grammar: + +```text +existing-variable-list := existing-variable + ((existing-variable | TO existing-variable))* +comparison-operator := "=" | "<" | "<=" | ">" | ">=" + | "NE" | "<>" | "~=" +predicate := comparison | "(" predicate ")" + | NOT predicate + | predicate AND predicate + | predicate OR predicate +open-range := LOWEST THRU finite-number + | finite-number THRU HIGHEST + | LOWEST THRU HIGHEST +``` + +Normatively require: + +- leading-star comments only when `*` is the first non-whitespace token at a command boundary, ending at the next command period; +- `COMMENT` command comments ending at their command period; +- non-nested `/* ... */` comments wherever whitespace is legal; unterminated or nested forms return `spss_syntax_error`; +- raw comments remain in LF-normalized source hashing and emit no operation; +- a comment-only program returns `spss_syntax_error` because plans cannot have zero operations; +- `TO` expands inclusively by current ordered input-schema state, not by variable spelling or numeric suffix; +- reverse ranges return `invalid_variable_range`; absent endpoints return `unknown_variable`; `TO` is forbidden for generated `INTO` target sequences; +- `NE`, `<>`, and `~=` lower to ordered `< OR >` comparisons; +- `NOT` is accepted only over the existing comparison/boolean grammar and lowers recursively with comparison complements and De Morgan's laws, preserving SQL three-valued truth and canonical same-operator flattening; +- open ranges use `ffefffffffffffff` for negative maximum finite binary64 and `7fefffffffffffff` for positive maximum finite binary64; SQL NULL/system missing is not included; +- `ADD VALUE LABELS` updates an existing typed value at its current ordinal and appends new values in source order, then emits one complete `replace_value_labels` operation per variable; +- comments, `TO`, grouped recodes and labels, open ranges, and additive labels remain in the Plan 0.1 output subset; grouped `FORMATS`/`VARIABLE LEVEL` and predicate aliases/`NOT` use existing Plan 0.2 operations. + +- [ ] **Step 5: Register Frontend 0.3 in the validator with an empty draft manifest.** + +Add `"0.3": "conformance/spss-syntax-frontend-0.3.json"` to +`FRONTEND_MANIFESTS`. Create the manifest in Task 5; for this task, keep the +schema/profile changes staged but do not commit until Task 5 supplies a valid +manifest and the repository is green. + +### Task 5: Define inherited compatibility and supersession + +**Files:** +- Create: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` +- Test: `conformance/spss-syntax-frontend-0.1.json` +- Test: `conformance/spss-syntax-frontend-0.2.json` + +**Interfaces:** +- Consumes: Task 4 request schema and profile. +- Produces: `inherited_manifests` semantics and effective-case expansion for Frontend 0.3. + +- [ ] **Step 1: Add failing inheritance tests.** + +```python +def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 2 + assert inventory.frontend_effective_cases["0.3"] == 57 + + +def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["inherited_manifests"][0]["manifest"] = "../outside.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="inherited manifest"): + validate_contract_artifacts(root) +``` + +- [ ] **Step 2: Create the initial Frontend 0.3 manifest.** + +Use this exact top-level shape: + +```json +{ + "manifest_version": "0.3", + "profile": "OpenStatSpec SPSS-like Syntax Frontend 0.3", + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "plan_contracts": [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2" + ], + "request_schema": "../transformation/spss-syntax-frontend-0.3.schema.json", + "plan_schemas": { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json" + }, + "inherited_manifests": [ + { + "manifest": "spss-syntax-frontend-0.1.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" + } + }, + { + "manifest": "spss-syntax-frontend-0.2.json", + "request_contract_override": "openstatspec-spss-syntax-frontend-v0.3", + "superseded_cases": {} + } + ], + "cases": [ + { + "id": "reject-comment-only-program", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* comment." + }, + "expected_source_hash": "8bfac9af8d0884b68cebc65a42360871592dd6e4e9cbead0bdf7029b6b64f4fc", + "expected_error": "spss_syntax_error" + }, + { + "id": "inline-block-comment-is-ignored", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "RECODE q1 (1 = 0) /* comment */." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "3c4ef459e9a2021100ef166ef0038c5f920b126d916db4c7273537d21b256f65", + "expected_error": null + } + ] +} +``` + +The initial effective count is 57: 11 unchanged Frontend 0.1 cases, all 44 +Frontend 0.2 cases, and two declared replacements. Expanded IDs are namespaced +as `0.1/` and `0.2/` so repeated IDs across manifests remain +distinct. + +- [ ] **Step 3: Implement inherited-case expansion and validation.** + +Add: + +```python +def _expand_frontend_cases( + version: str, + manifest: Mapping[str, object], + manifests: Mapping[str, Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + """Return inherited cases with contract override followed by declared cases.""" +``` + +Require inherited paths to be canonical filenames in `conformance/`, target +older registered Frontend manifests only, and use the current manifest's exact +contract as `request_contract_override`. Deep-copy each inherited case and +replace only `request.contract`; source text, expected source hash, plan, +expected plan hash, and expected diagnostic remain unchanged. + +`superseded_cases` maps an inherited case ID to a declared 0.3 replacement case +ID. Require every old ID to exist, every replacement ID to exist in declared +cases, and the replacement request to be identical to the inherited request +except for the current request contract. This preserves `input_alias`, +`input_schema`, and `source_text`; the replacement may change the expected +diagnostic and expected plan fields to document the new behavior. Exclude +superseded old cases from the effective set. Reject inheritance cycles and +duplicate expanded IDs. + +- [ ] **Step 4: Run the focused tests and complete Task 4/5 commit.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend declared counts are `13/44/2`; Frontend effective counts are `13/44/57`. + +Commit: + +```bash +git add transformation/spss-syntax-frontend-0.3.schema.json docs/spss-syntax-frontend-profile-0.3.md conformance/spss-syntax-frontend-0.3.json src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "spec: define SPSS frontend 0.3 contract" +``` + +### Task 6: Add comments, variable lists, and grouped forms + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Consumes: Task 5 inheritance and supersession format. +- Produces: 14 declared cases covering comments and schema-ordered variable-list expansion. + +- [ ] **Step 1: Add a failing exact-count and supersession test.** + +```python +def test_frontend_03_comment_and_varlist_case_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 14 + assert inventory.frontend_effective_cases["0.3"] == 69 +``` + +The effective count is `13 + 44 - 2 superseded + 14 = 69`. + +- [ ] **Step 2: Add the four remaining exact comment cases.** + +Add these case IDs and outcomes: + +```text +comment-command-before-recode success, exact Plan 0.1 recode +leading-star-comment-before-recode success, same Plan 0.1 recode +comments-change-source-hash-not-plan-hash success, same plan with a distinct source hash +reject-unterminated-inline-comment spss_syntax_error +``` + +For each success, embed or reference an exact existing Plan 0.1 case and include +an independently calculated lowercase source hash. A representative case is: + +```json +{ + "id": "leading-star-comment-before-recode", + "request": { + "contract": "openstatspec-spss-syntax-frontend-v0.3", + "input_alias": "parent", + "input_schema": {"variables": [{"name": "q1", "storage_kind": "numeric"}]}, + "source_text": "* recode one to zero.\nRECODE q1 (1 = 0)." + }, + "expected_plan_contract": "openstatspec-transformation-plan-v0.1", + "expected_plan_case_0_1": "in-place-recode-default-copy", + "expected_plan_hash": "091a2daeb490deb3369cba83b5197d1c43767432c9a04cbf3e55bd69296b67a8", + "expected_source_hash": "dddbaf0dd69f364ccc895d0d3b65daa63208edfd9b4e8e2112f915f1e73e5d70", + "expected_error": null +} +``` + +Calculate each source hash with: + +```bash +python3 -c 'from openstatspec_specification.artifacts import source_hash; print(source_hash("* recode one to zero.\nRECODE q1 (1 = 0)."))' +``` + +Write the printed lowercase digest into the manifest; do not calculate hashes at adapter runtime. + +- [ ] **Step 3: Verify the two released comment failures remain explicitly superseded.** + +Confirm Frontend 0.1 inheritance metadata remains: + +```json +"superseded_cases": { + "reject-comment-command": "reject-comment-only-program", + "reject-inline-comment": "inline-block-comment-is-ignored" +} +``` + +The replacement cases use the exact old source text. The first remains a +failure but changes from unsupported command to empty-program syntax error; the +second becomes a success. Do not edit the released 0.1 manifest. + +- [ ] **Step 4: Add eight variable-list/group cases.** + +Add these exact IDs: + +```text +recode-to-expands-dictionary-order +value-labels-to-expands-dictionary-order +formats-grouped-to-expands-source-order +variable-level-to-expands-source-order +to-sees-variable-created-by-preceding-command +reject-reversed-to-range +reject-unknown-to-endpoint +reject-to-generated-into-targets +``` + +Use input order `a, middle, c, outside` and prove `a TO c` expands to exactly +`a, middle, c`, regardless of spelling. `FORMATS a TO c (F8.2)` emits three +ordered `set_format` operations. `VARIABLE LEVEL a TO c (NOMINAL)` emits three +ordered `set_measurement_level` operations. `COMPUTE appended = a.` followed by +`FORMATS c TO appended (F8.2).` proves current command-order schema state. + +Use stable errors: + +```text +reversed existing-variable range -> invalid_variable_range +absent endpoint -> unknown_variable +TO in generated INTO targets -> spss_syntax_error +``` + +- [ ] **Step 5: Validate all explicit source and plan hashes.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend 0.3 declared/effective counts are `14/69`; no Plan manifest count or hash changes. + +- [ ] **Step 6: Commit comments and variable-list fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py +git commit -m "spec: add frontend comments and variable lists" +``` + +### Task 7: Add predicate aliases and open recode ranges + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` + +**Interfaces:** +- Consumes: unchanged Plan 0.1 range nodes and Plan 0.2 comparison/boolean nodes. +- Produces: 14 declared cases for not-equal/NOT lowering and finite open-range lowering. + +- [ ] **Step 1: Add a failing exact-count test.** + +```python +def test_frontend_03_predicate_and_open_range_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 28 + assert inventory.frontend_effective_cases["0.3"] == 83 +``` + +- [ ] **Step 2: Add eight exact predicate cases.** + +```text +ne-keyword-lowers-to-less-or-greater +angle-not-equal-lowers-to-less-or-greater +tilde-not-equal-lowers-to-less-or-greater +not-equality-lowers-to-less-or-greater +not-nested-and-lowers-by-de-morgan +double-not-retains-canonical-comparison +reject-not-numeric-truthiness +reject-chained-comparison +``` + +Use this exact canonical lowering table in profile and cases: + +```text +NOT (left = right) -> (left < right OR left > right) +NOT (left < right) -> left >= right +NOT (left <= right) -> left > right +NOT (left > right) -> left <= right +NOT (left >= right) -> left < right +NOT (a AND b) -> NOT a OR NOT b +NOT (a OR b) -> NOT a AND NOT b +``` + +`NE`, `<>`, and `~=` all emit the same ordered n-ary `or` node containing `<` +then `>`. Apply existing maximal same-operator flattening after recursive +lowering. Do not constant-fold or reorder. Fail numeric truthiness such as +`IF (NOT source_a) ...` with `expression_type_unsupported`; fail +`a < b < c` with `spss_syntax_error`. + +- [ ] **Step 3: Add six exact open-range cases.** + +```text +recode-lowest-thru-upper +recode-lower-thru-highest +recode-lowest-thru-highest +reject-string-open-range +reject-highest-as-lower-endpoint +reject-lowest-as-upper-endpoint +``` + +Expected finite endpoints are exact typed values: + +```json +{"type": "binary64", "bits": "ffefffffffffffff"} +{"type": "binary64", "bits": "7fefffffffffffff"} +``` + +String open ranges fail `type_mismatch`. `HIGHEST THRU 1` and +`1 THRU LOWEST` fail `spss_syntax_error`. System missing remains unmatched +unless another explicit rule handles `SYSMIS`. + +- [ ] **Step 4: Run exact identity gates.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 tools/validate_repository.py +``` + +Expected: Frontend 0.3 declared/effective counts are `28/83`; all Plan 0.1/0.2 hashes remain unchanged. + +- [ ] **Step 5: Commit predicate and range fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py docs/spss-syntax-frontend-profile-0.3.md +git commit -m "spec: add frontend predicates and open ranges" +``` + +### Task 8: Add additive value-label semantics + +**Files:** +- Modify: `conformance/spss-syntax-frontend-0.3.json` +- Modify: `tests/test_contract_artifacts.py` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` + +**Interfaces:** +- Consumes: Frontend request `input_schema.variables[].value_labels` and existing Plan `replace_value_labels`. +- Produces: seven declared cases and deterministic ordered merge semantics. + +- [ ] **Step 1: Add the final failing inventory test.** + +```python +def test_frontend_03_final_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} +``` + +- [ ] **Step 2: Add seven exact additive-label cases.** + +```text +add-value-labels-preserves-existing-and-appends +add-value-labels-replaces-existing-label-in-place +value-labels-then-add-value-labels-uses-ordered-state +add-value-labels-multiple-variable-group +add-string-value-labels +reject-add-value-labels-duplicate-source-value +reject-add-value-labels-type-mismatch +``` + +Use this normative merge algorithm: + +```python +def merge_labels(existing, additions): + result = list(existing) + index = {canonical_typed_value(item["value"]): position for position, item in enumerate(result)} + seen_additions = set() + for addition in additions: + key = canonical_typed_value(addition["value"]) + if key in seen_additions: + raise duplicate_value_label + seen_additions.add(key) + if key in index: + result[index[key]] = addition + else: + index[key] = len(result) + result.append(addition) + return result +``` + +This is specification pseudocode, not an implementation API. Typed values use +their exact canonical identity; numeric positive zero canonicalization and +string exactness remain inherited. Emit one complete `replace_value_labels` +operation per variable in group/source order. A preceding `VALUE LABELS` +replaces state; a following `ADD VALUE LABELS` merges against that replacement, +not against the original request. + +- [ ] **Step 3: Prove Plan 0.1 output selection.** + +At least four successful additive-label cases contain no Plan 0.2 operation and +must declare: + +```json +"expected_plan_contract": "openstatspec-transformation-plan-v0.1" +``` + +Their expected plans contain only complete ordered `replace_value_labels` +operations and validate against `plan-0.1.schema.json`. + +- [ ] **Step 4: Run the final manifest inventory and identity checks.** + +Run: + +```bash +python3 -m pytest tests/test_contract_artifacts.py -q +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: Plan `4/26`, Frontend declared `13/44/35`, Frontend effective +`13/44/90`, Binding `6/11`; all hashes and references validate. + +- [ ] **Step 5: Commit additive-label fixtures.** + +```bash +git add conformance/spss-syntax-frontend-0.3.json tests/test_contract_artifacts.py docs/spss-syntax-frontend-profile-0.3.md +git commit -m "spec: define additive value label syntax" +``` + +### Task 9: Complete examples, release notes, and adapter handoff + +**Files:** +- Create: `examples/spss-syntax-transformation-0.3.md` +- Modify: `README.md` +- Modify: `CHANGELOG.md` +- Modify: `ROADMAP.md` +- Modify: `docs/spss-frontend-roadmap.md` +- Modify: `docs/spss-syntax-frontend-profile-0.3.md` +- Modify: `src/openstatspec_specification/artifacts.py` +- Modify: `tests/test_contract_artifacts.py` + +**Interfaces:** +- Consumes: complete 0.3 schema/profile/manifest and validated counts. +- Produces: specification release-candidate documentation and exact downstream adapter claim gates. + +- [ ] **Step 1: Add failing documentation presence checks.** + +```python +def test_frontend_03_release_candidate_documentation_is_complete() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") + example = (ROOT / "examples/spss-syntax-transformation-0.3.md").read_text(encoding="utf-8") + assert "spss-syntax-frontend-0.3.schema.json" in readme + assert "SPSS Syntax Frontend 0.3" in changelog + assert "Frontend 0.3" in roadmap + assert "ADD VALUE LABELS" in example + assert "Plan 0.1/0.2" in example +``` + +- [ ] **Step 2: Write one complete 0.3 example.** + +Use one exact program whose corresponding manifest cases already prove each +production independently: + +```spss +* Normalize the selected score block. +RECODE score_a TO score_c (LOWEST THRU 0 = 0) (1 THRU HIGHEST = 1). +ADD VALUE LABELS score_a TO score_c 0 'Non-positive' 1 'Positive'. +FORMATS score_a TO score_c (F1.0). +IF (NOT (score_a = score_b)) mismatch = 1. +EXECUTE. +``` + +The example must state that `mismatch` is an existing numeric target, explain +dictionary-order `TO`, finite open-range lowering, complete additive-label +replacement, `< OR >` lowering, source hashing with comments, Plan 0.1/0.2 +selection, and unchanged SQL/Dolt binding policy. + +- [ ] **Step 3: Update the artifact index and Unreleased changelog.** + +In `README.md`, list the 0.3 profile, schema, and conformance manifest beside +0.1/0.2. In `CHANGELOG.md` under Unreleased record: + +- Frontend 0.3 is a release candidate, not a published stable claim. +- It adds comments, existing-variable `TO`, grouped forms, predicate aliases and + bounded `NOT`, open recode ranges, and additive value labels. +- It emits only immutable Plan 0.1/0.2 and adds no SQL binding operation. +- Repository validation now covers all Plan/Frontend/Binding schemas, + manifests, references, and hashes. + +- [ ] **Step 4: Mark only completed roadmap milestones.** + +In `ROADMAP.md` section 7, mark the Frontend 0.3 specification item complete +only after schema, profile, 35 declared cases, inherited compatibility, and +repository validation are all present. Leave adapter implementation and all +later expression/schema/case/group milestones unchecked. + +In `docs/spss-frontend-roadmap.md`, mark Milestone 0 and Milestone 1 as +specification-complete while stating that adapter claims remain pending and +separately gated. + +- [ ] **Step 5: Add exact adapter handoff gates to the 0.3 profile.** + +Require an adapter claim to: + +1. pin the exact future specification release commit; +2. accept only the exact Frontend 0.3 request contract for this suite; +3. run all 90 effective Frontend 0.3 cases; +4. preserve every inherited Plan 0.1/0.2 object and hash; +5. pass all 35 declared cases with exact source/plan hashes and diagnostics; +6. continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites; +7. keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy unchanged. + +State explicitly that adapter evidence does not block specification publication. + +- [ ] **Step 6: Promote the 0.3 profile from draft to release candidate.** + +Only after every prior task is green, change its status line to: + +```markdown +Status: release candidate for the next OpenStatSpec minor release. This profile +is not published stable until a protected or signed specification tag targets +its exact reviewed commit and tag-context CI passes. +``` + +Do not choose the final specification release number in this implementation +task; release maintainers select it from the complete set of included normative +changes. + +- [ ] **Step 7: Run final specification verification.** + +Run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +git diff --check +git status --short +``` + +Expected: all pytest tests pass, validator reports Plan `4/26`, Frontend +declared `13/44/35`, Frontend effective `13/44/90`, Binding `6/11`, diff check +is clean, and status lists only intended release-candidate documentation. + +- [ ] **Step 8: Commit release-candidate documentation.** + +```bash +git add examples/spss-syntax-transformation-0.3.md README.md CHANGELOG.md ROADMAP.md docs/spss-frontend-roadmap.md docs/spss-syntax-frontend-profile-0.3.md src/openstatspec_specification/artifacts.py tests/test_contract_artifacts.py +git commit -m "docs: prepare SPSS frontend 0.3 candidate" +``` + +### Task 10: Review and publish the specification branch + +**Files:** +- Verify: all files changed by Tasks 1-9 + +**Interfaces:** +- Consumes: complete specification candidate and green repository checks. +- Produces: reviewable branch/PR; no specification tag and no adapter claim. + +- [ ] **Step 1: Review the complete contract diff.** + +Run: + +```bash +git status --short --branch +git diff origin/main...HEAD --stat +git diff origin/main...HEAD -- transformation docs conformance src tests tools README.md CHANGELOG.md ROADMAP.md pyproject.toml .github/workflows/ci.yml +git log --oneline origin/main..HEAD +``` + +Confirm no published 0.1/0.2 schema or manifest changed by running: + +```bash +git diff --exit-code origin/main...HEAD -- \ + transformation/plan-0.1.schema.json \ + transformation/plan-0.2.schema.json \ + transformation/spss-syntax-frontend-0.1.schema.json \ + transformation/spss-syntax-frontend-0.2.schema.json \ + conformance/transformation-plan-0.1.json \ + conformance/transformation-plan-0.2.json \ + conformance/spss-syntax-frontend-0.1.json \ + conformance/spss-syntax-frontend-0.2.json \ + conformance/in-place-transformation-0.1.json \ + conformance/in-place-transformation-0.2.json +``` + +Expected: exit zero and no diff for every published artifact. + +- [ ] **Step 2: Run the final clean-checkout gate.** + +Create an isolated clean checkout of `HEAD`, install `.[dev]`, and run: + +```bash +python3 -m pytest -q +python3 tools/validate_repository.py +``` + +Expected: all checks pass without depending on sibling adapter repositories. + +- [ ] **Step 3: Push and create a specification PR.** + +```bash +git push -u origin agent/spss-frontend-03 +gh pr create --repo OpenStatSpec/specification \ + --base main \ + --head agent/spss-frontend-03 \ + --title "Define SPSS syntax frontend 0.3" \ + --body $'Defines the bounded SPSS Syntax Frontend 0.3 contract.\n\n- Emits only immutable Plan 0.1/0.2 objects.\n- Adds 35 declared and 90 effective frontend cases.\n- Preserves all published 0.1/0.2 artifacts and hashes.\n- Adds repository-owned schema, reference, and hash validation.\n\nVerification: `python3 -m pytest -q` and `python3 tools/validate_repository.py`.' +``` + +The PR body must list contract boundaries, exact declared/effective case counts, +immutable older-artifact verification, validator/test commands, and downstream +adapter gates. Do not create a specification tag or GitHub release in this task. diff --git a/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md new file mode 100644 index 0000000..445f224 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-spss-frontend-expansion-design.md @@ -0,0 +1,193 @@ +# SPSS Frontend Expansion Design + +## Goal + +Expand the bounded SPSS syntax frontend in independently claimable stages +without promising general IBM SPSS Statistics compatibility, changing published +Plan 0.1/0.2 identities, or weakening the in-place and Dolt ownership +contracts. + +## Research basis + +The design compares the current Frontend 0.2 grammar and conformance artifacts +with IBM's command reference and focused data-management references from GNU +PSPP and UCLA OARC. The highest-value gaps are comments, variable-list syntax, +ordinary predicates, arithmetic, missing-value handling, deterministic +functions, string preparation, and selected dictionary operations. + +The current plan vocabulary creates a hard version boundary. Comments, +variable-list expansion, alternate predicate syntax, open-ended recode ranges, +and additive value-label syntax can lower to unchanged Plan 0.1/0.2 operations. +Arithmetic, functions, missing predicates, string expressions, schema changes, +case deletion, ordering, grouping, and joins require new normative semantics. + +Primary references: + +- [IBM SPSS Statistics Command Syntax Reference](https://www.ibm.com/docs/en/SSLVMB_25.0.0/pdf/en/IBM_SPSS_Statistics_Command_Syntax_Reference.pdf) +- [IBM COMMENT command](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=comment-overview-command) +- [IBM string functions](https://www.ibm.com/docs/en/spss-statistics/25.0.0?topic=expressions-string-functions) +- [IBM SELECT IF](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-select-if) +- [IBM FORMATS syntax rules](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=formats-syntax-rules-command) +- [IBM date and time arithmetic](https://www.ibm.com/docs/SSLVMB_sub/statistics_reference_project_ddita/spss/base/syn_date_and_time_arithmetic_operations_date_time_variables.html) +- [IBM Aggregate Data](https://www.ibm.com/docs/en/spss-statistics/30.0.0?topic=transformations-aggregate-data) +- [IBM MATCH FILES](https://www.ibm.com/docs/en/spss-statistics/32.0.0?topic=reference-match-files) + +Secondary references are the [GNU PSPP manual](https://www.gnu.org/software/pspp/manual/) +and [UCLA OARC SPSS resources](https://stats.oarc.ucla.edu/spss/). + +## Selected approach + +Use a contract staircase instead of one broad release. + +1. Frontend 0.3 accepts only syntax that deterministically emits an exact Plan + 0.1 or Plan 0.2 object. +2. A later Plan, Frontend, and In-Place Binding release adds typed numeric + expressions and dictionary metadata together. +3. String and physical-schema operations form a separate contract milestone + with independent SQL atomicity evidence. +4. Case-count/order operations and group/cross-dataset operations use separate + profiles rather than stretching the current same-cases in-place contract. + +This approach creates useful progress before SQL executor changes, keeps old +canonical bytes and hashes immutable, and prevents experimental adapter +identifiers from becoming de facto specification contracts. + +## Contract architecture + +Every accepted syntax expansion increments the frontend contract because the +contract selects accepted grammar, diagnostics, source spans, and output-plan +selection. A frontend-only release may still emit older plan contracts. + +A new Plan contract is required when an addition introduces a new operation, +expression shape, type rule, evaluation rule, or meaning for an existing +operation. A new In-Place Binding contract is required when SQL execution, +catalog mutation, audit behavior, rollback, or profile capability rules change. + +Older schemas, fixtures, canonical JSON, hashes, and diagnostics remain +immutable. New manifests inherit successful and failing older cases and prove +that old source continues to emit the same old plans. An inherited case is +identical in the effective set, apart from its namespaced ID and request +contract override, unless its ID appears in `superseded_cases`. A superseding +replacement must use the same request input alias, input schema, and source +text; it may change only the expected diagnostic and expected plan fields that +describe the revised frontend behavior. + +## Milestones + +### 0. Specification integrity and classification + +- Align release-status wording across roadmap, profile documents, changelog, + tags, and releases. +- Extend repository validation to cover Plan, Frontend, and In-Place manifests, + schemas, unique IDs, references, canonical bytes, and hashes. +- Publish a command-classification table assigning each proposed SPSS form to + Frontend-only, new Plan/Binding, a separate profile, or explicit non-goal. + +### 1. Frontend 0.3: syntax depth over Plan 0.1/0.2 + +- Command comments (`COMMENT` and leading `*`) and inline block comments. +- Existing-variable lists and dictionary-order `TO` expansion. +- Grouped forms of already supported labels, formats, levels, and recodes. +- `NOT`, `NE`, `<>`, and `~=` over the existing comparison/boolean grammar, + with explicit three-valued semantics and canonical lowering to existing + predicate nodes. +- `LOWEST`/`HIGHEST` recode ranges with finite canonical endpoints. +- `ADD VALUE LABELS`, compiled to a complete ordered + `replace_value_labels` operation using ordered metadata state initialized + from the request and updated by preceding commands. + +Frontend 0.3 lists only Plan 0.1 and Plan 0.2 as outputs. It does not add an +executor operation or change an in-place binding. + +### 2. Typed numeric expressions and dictionary metadata + +- Parentheses, unary minus, `+`, `-`, `*`, and `/` with defined precedence, + overflow, division, and system-missing behavior. +- A deterministic numeric-function whitelist, initially `ABS`, `MOD`, `TRUNC`, + `RND`, `SQRT`, `LN`, `LG10`, `EXP`, `MIN`, `MAX`, `SUM`, and `MEAN`. +- `MISSING`, `SYSMIS`, `NMISS`, and `NVALID`, distinguishing system missing + from metadata-defined user missing. +- `MISSING VALUES` as a catalog-only operation. +- Numeric output formats `E`, `COMMA`, `DOT`, and `PCT`; locale-dependent + currency formats remain excluded. +- Deterministic date construction, extraction, arithmetic, and display based on + SPSS numeric serial values; current-time and locale-ambiguous parsing remain + excluded. +- Restricted `DO IF`/`ELSE IF`/`ELSE` containing only operations in the same + contract. A branch condition is evaluated once against the row state at block + entry, while selected body operations retain their source order. + +The Plan expression IR is typed and source-neutral. Adapters must not insert +source expressions into backend SQL or inherit backend coercion, function, +locale, collation, or error behavior. + +### 3. String expressions and schema operations + +- `NUMERIC` and `STRING` declarations with exact initialization and width + semantics. +- String assignment and comparison with explicit fixed-width, blank, Unicode, + padding, truncation, and collation rules. +- A deterministic string-function whitelist: `CONCAT`, `UPCASE`, `LOWER`, + `LTRIM`, `RTRIM`, `LENGTH`, `SUBSTR`, `INDEX`, and `REPLACE`. +- Simultaneous `RENAME VARIABLES` with collision and case-only rename rules. +- Controlled `DELETE VARIABLES` with catalog-reference cleanup and dependency + validation. + +Each SQL profile must publish add, rename, and drop-column atomicity evidence. +MySQL, MariaDB, and Dolt continue to require pre-provisioned new targets where +native add-column DDL cannot satisfy one-transaction apply. Rename and delete +must fail before mutation on a profile whose native DDL cannot satisfy the +contract; an output/staging table or full-table copy is not an allowed +workaround. Dolt commits remain caller-owned. + +### 4. Case operations + +- `SELECT IF` is a destructive, row-count-changing operation with exact TRUE, + FALSE, and UNKNOWN behavior and explicit affected-case evidence. +- `SORT CASES` is deferred until the standard defines persistent logical case + order, stable tie-breaking, missing ordering, and string collation. Physical + table order is never treated as canonical case order. + +These operations use a separate case-transformation profile because the current +in-place binding preserves case count and order. + +### 5. Group and cross-dataset operations + +- Consider only `AGGREGATE OUTFILE=* MODE=ADDVARIABLES` with a small, + deterministic function set and unchanged row count. +- Consider a restricted keyed lookup with one active dataset and one unique-key + lookup dataset, explicit selected columns, and unchanged active row count. + +Full `AGGREGATE OUTFILE`, unrestricted `MATCH FILES`, and any hidden output, +staging, snapshot, rollback, or copied dataset remain forbidden. Dolt is the +sole persistent version/history layer for successful edits. + +## Explicit non-goals + +- No claim of full SPSS syntax compatibility. +- No statistical procedures, procedure output, weighting, `SPLIT FILE`, or + session-wide analysis state. +- No `TEMPORARY` or `FILTER` until a real procedure/query scope exists. +- No random, current-time, `LAG`, cumulative, locale-dependent, or other + order/state-dependent functions in the deterministic expression profile. +- No macros, loops, vectors, extension-language execution, unrestricted joins, + reshaping, or `ALTER TYPE` in these milestones. +- No persistent output/staging copies or OpenStatSpec-managed rollback state. + +## Conformance and release gates + +Each milestone provides a profile document, request/plan schema where +applicable, positive and failure manifest cases, exact source/plan hashes, +stable diagnostics and spans, and inherited compatibility cases. + +Specification validation checks artifact structure and identity. Adapter claims +remain downstream: each adapter pins the exact specification commit and runs +the complete applicable frontend/plan suites. Every binding claim also runs +SQLite plus every claimed PostgreSQL, MySQL, MariaDB, Dolt, or future dialect +service profile, including rollback and forbidden-artifact cases. + +Specification publication precedes adapter claims and does not wait for them. +No adapter may claim an official contract identifier before that contract is +published from an immutable specification commit. Experimental adapter syntax +uses an explicitly adapter-owned identifier until publication and is not +interoperability evidence for the future official contract. diff --git a/docs/transformation-plan-profile-0.2.md b/docs/transformation-plan-profile-0.2.md index f3380fc..2340ee1 100644 --- a/docs/transformation-plan-profile-0.2.md +++ b/docs/transformation-plan-profile-0.2.md @@ -2,7 +2,8 @@ ## Status and scope -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. This optional profile is the backward-compatible successor to Transformation Plan 0.1. Its contract identifier is diff --git a/docs/transformation-plan-sql-binding-0.2.md b/docs/transformation-plan-sql-binding-0.2.md index b268c87..4c8c835 100644 --- a/docs/transformation-plan-sql-binding-0.2.md +++ b/docs/transformation-plan-sql-binding-0.2.md @@ -2,7 +2,8 @@ ## Status -Status: release candidate for the planned OpenStatSpec `v0.3.0` release. This profile is not a published stable specification until a `v0.3.0` tag targets its exact commit. +Status: released in OpenStatSpec `v0.3.0` at immutable specification commit +`cd8f198c68b849eb8ed018a894670a0904c2181d`. ## Normative decision diff --git a/examples/spss-syntax-transformation-0.3.md b/examples/spss-syntax-transformation-0.3.md new file mode 100644 index 0000000..9648389 --- /dev/null +++ b/examples/spss-syntax-transformation-0.3.md @@ -0,0 +1,41 @@ +# SPSS-like transformation frontend 0.3 release-candidate example + +This exact program combines the Frontend 0.3 syntax additions. The input +schema has `score_a`, `score_b`, and `score_c` as numeric variables in that +dictionary order. `mismatch` is an existing numeric target; the program does +not create it. + +```spss +* Normalize the selected score block. +RECODE score_a TO score_c (LOWEST THRU 0 = 0) (1 THRU HIGHEST = 1). +ADD VALUE LABELS score_a TO score_c 0 'Non-positive' 1 'Positive'. +FORMATS score_a TO score_c (F1.0). +IF (NOT (score_a = score_b)) mismatch = 1. +EXECUTE. +``` + +`TO` expands inclusively in dictionary order, so each command addresses +`score_a`, `score_b`, and `score_c`, not names inferred from a suffix. The +finite open ranges lower `LOWEST` and `HIGHEST` to the finite binary64 +endpoints defined by the profile; SQL NULL/system-missing remains outside the +range. `ADD VALUE LABELS` lowers to a complete additive-label replacement for +each variable, preserving the ordered existing label state and applying the +updates. `NOT (score_a = score_b)` lowers to the existing `< OR >` comparison +form, with SQL three-valued truth preserved. + +The leading comment is retained in the LF-normalized source used for the +source SHA-256 hash, while comments emit no operation. The full emitted plan +has its own canonical plan hash over the immutable Plan 0.1/0.2 outputs. The recode and labels use the Plan 0.1 +operation subset, while grouped `FORMATS`, the conditional assignment, and its +lowered predicate use Plan 0.2 operations. A program without any Plan 0.2-only +operation selects Plan 0.1 instead. The corresponding manifest cases prove +each production independently, including +`recode-lowest-thru-upper`, `recode-lower-thru-highest`, +`add-value-labels-multiple-variable-group`, `formats-grouped-to-expands-source-order`, +and `not-equality-lowers-to-less-or-greater`. + +Frontend 0.3 adds no SQL binding operation. The existing same-dataset and +same-physical-table policy is unchanged. MySQL, MariaDB, and Dolt still +require caller-owned pre-provisioning for a new target, and Dolt still keeps +branch, HEAD, working-set, and commit ownership with the caller; the frontend +does not call `DOLT_COMMIT`. diff --git a/pyproject.toml b/pyproject.toml index 54d5d8d..e4f844a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,9 +10,10 @@ readme = "README.md" requires-python = ">=3.11" license = {text = "Apache-2.0"} authors = [{name = "OpenStatSpec"}] +dependencies = ["jsonschema>=4.23,<5"] [project.optional-dependencies] -dev = ["pytest>=8"] +dev = ["pytest>=8,<9"] [tool.hatch.build.targets.wheel] packages = ["src/openstatspec_specification"] diff --git a/src/openstatspec_specification/artifacts.py b/src/openstatspec_specification/artifacts.py new file mode 100644 index 0000000..052ed39 --- /dev/null +++ b/src/openstatspec_specification/artifacts.py @@ -0,0 +1,904 @@ +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping + +from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError, ValidationError + + +PLAN_MANIFESTS = { + "0.1": "conformance/transformation-plan-0.1.json", + "0.2": "conformance/transformation-plan-0.2.json", +} +FRONTEND_MANIFESTS = { + "0.1": "conformance/spss-syntax-frontend-0.1.json", + "0.2": "conformance/spss-syntax-frontend-0.2.json", + "0.3": "conformance/spss-syntax-frontend-0.3.json", +} +BINDING_MANIFESTS = { + "0.1": "conformance/in-place-transformation-0.1.json", + "0.2": "conformance/in-place-transformation-0.2.json", +} +V030_COMMIT = "cd8f198c68b849eb8ed018a894670a0904c2181d" +V030_PROFILE_DOCUMENTS = ( + "docs/transformation-plan-profile-0.2.md", + "docs/spss-syntax-frontend-profile-0.2.md", + "docs/transformation-plan-sql-binding-0.2.md", +) +FRONTEND_03_PROFILE_DOCUMENT = "docs/spss-syntax-frontend-profile-0.3.md" + + +class ArtifactValidationError(ValueError): + pass + + +@dataclass(frozen=True) +class ArtifactInventory: + plan_cases: Mapping[str, int] + frontend_declared_cases: Mapping[str, int] + frontend_effective_cases: Mapping[str, int] + binding_cases: Mapping[str, int] + + +def canonical_json_bytes(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def source_hash(source_text: str) -> str: + normalized = source_text.replace("\r\n", "\n").replace("\r", "\n") + return _sha256(normalized.encode("utf-8")) + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ArtifactValidationError(message) + + +def _canonical_parts(relative: str) -> list[str]: + _require( + isinstance(relative, str) + and not relative.startswith("/") + and "\\" not in relative, + f"noncanonical artifact path: {relative}", + ) + parts = relative.split("/") + _require( + all(part not in {"", ".", ".."} for part in parts), + f"noncanonical artifact path: {relative}", + ) + return parts + + +def _safe_candidate(root: Path, relative: str) -> Path: + parts = _canonical_parts(relative) + candidate = root.joinpath(*parts) + current = root + try: + for part in parts: + current = current / part + _require( + not current.is_symlink(), + f"artifact path traverses symlink: {relative}", + ) + except OSError as error: + raise ArtifactValidationError( + f"artifact path cannot be inspected: {relative} ({type(error).__name__})" + ) from None + return candidate + + +def _load_json(root: Path, relative: str) -> object: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + return json.loads(candidate.read_text(encoding="utf-8")) + except ArtifactValidationError: + raise + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + + +def _load_text(root: Path, relative: str) -> str: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + return candidate.read_text(encoding="utf-8") + except ArtifactValidationError: + raise + except (OSError, UnicodeError) as error: + raise ArtifactValidationError( + f"artifact cannot be decoded: {relative} ({type(error).__name__})" + ) from None + + +def _require_file(root: Path, relative: str) -> None: + candidate = _safe_candidate(root, relative) + try: + _require(candidate.is_file(), f"artifact is missing: {relative}") + except OSError as error: + raise ArtifactValidationError( + f"artifact cannot be inspected: {relative} ({type(error).__name__})" + ) from None + + +def _resolve_manifest_reference(manifest_relative: str, reference: str) -> str: + _require( + isinstance(reference, str) + and bool(reference) + and not reference.startswith("/") + and "\\" not in reference, + f"noncanonical manifest reference: {reference!r}", + ) + resolved: list[str] = [] + candidate = PurePosixPath(manifest_relative).parent / reference + for part in candidate.parts: + if part in {"", "."}: + continue + if part == "..": + _require(bool(resolved), f"manifest reference escapes repository root: {reference}") + resolved.pop() + continue + resolved.append(part) + _require(bool(resolved), f"manifest reference resolves to repository root: {reference}") + return "/".join(resolved) + + +def _require_unique_case_ids(manifest: Mapping[str, object], context: str) -> None: + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{context}: cases must be a list") + seen: set[str] = set() + for case in cases: + _require(isinstance(case, dict), f"{context}: each case must be an object") + case_id = case.get("id") + _require( + isinstance(case_id, str) and bool(case_id), + f"{context}: case id must be a non-empty string", + ) + _require(case_id not in seen, f"{context}: duplicate case id: {case_id}") + seen.add(case_id) + + +def _schema(root: Path, relative: str) -> Mapping[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: schema must be an object") + try: + Draft202012Validator.check_schema(value) + except SchemaError as error: + raise ArtifactValidationError( + f"{relative}: invalid JSON Schema: {error.message}" + ) from None + return value + + +def _load_manifest(root: Path, relative: str, version: str, contract: str) -> dict[str, object]: + value = _load_json(root, relative) + _require(isinstance(value, dict), f"{relative}: manifest must be an object") + _require( + value.get("manifest_version") == version, + f"{relative}: manifest_version must be {version}", + ) + _require( + value.get("contract") == contract, + f"{relative}: contract must be {contract}", + ) + _require_unique_case_ids(value, relative) + return value + + +def _expected_error(case: Mapping[str, object], context: str) -> str | None: + value = case.get("expected_error") + _require( + value is None or (isinstance(value, str) and bool(value)), + f"{context}: expected_error must be null or a non-empty string", + ) + return value if isinstance(value, str) else None + + +def _schema_is_valid(value: object, schema: Mapping[str, object]) -> bool: + try: + Draft202012Validator(schema).validate(value) + except ValidationError: + return False + return True + + +def _validate_plan_manifests( + root: Path, + manifests: Mapping[str, Mapping[str, object]], +) -> Mapping[str, Mapping[str, object]]: + """Return version -> case-id -> successful canonical plan case.""" + successful: dict[str, Mapping[str, object]] = {} + for version, manifest in manifests.items(): + manifest_relative = PLAN_MANIFESTS[version] + schema_reference = manifest.get("schema") + _require( + isinstance(schema_reference, str), + f"{manifest_relative}: schema must be a string", + ) + schema = _schema(root, _resolve_manifest_reference(manifest_relative, schema_reference)) + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + successful_cases: dict[str, Mapping[str, object]] = {} + validator = Draft202012Validator(schema) + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + expected_error = _expected_error(case, context) + plan = case.get("plan") + _require(isinstance(plan, dict), f"{context}.plan: must be an object") + expected_hash = case.get("expected_plan_hash") + if expected_error is not None: + _require( + "expected_plan_hash" not in case or expected_hash is None, + f"{context}: failing case must omit expected_plan_hash or set it to null", + ) + try: + validator.validate(plan) + except ValidationError: + _require( + expected_error == "plan_schema_invalid", + f"{context}: semantic failure must have a schema-valid plan", + ) + else: + _require( + expected_error != "plan_schema_invalid", + f"{context}: plan_schema_invalid requires a schema-invalid plan", + ) + continue + + try: + validator.validate(plan) + except ValidationError as error: + raise ArtifactValidationError( + f"{context}.plan: schema validation failed: {error.message}" + ) from None + _require( + plan.get("contract") == manifest.get("contract"), + f"{context}.plan: contract does not match manifest", + ) + actual_hash = _sha256(canonical_json_bytes(plan)) + _require( + isinstance(expected_hash, str) and expected_hash == actual_hash, + f"{context}: plan hash mismatch", + ) + successful_cases[case["id"]] = case + successful[version] = successful_cases + return successful + + +def _frontend_plan_schemas( + root: Path, + version: str, + manifest: Mapping[str, object], +) -> Mapping[str, Mapping[str, object]]: + manifest_relative = FRONTEND_MANIFESTS[version] + if version == "0.1": + contract = manifest.get("plan_contract") + reference = manifest.get("plan_schema") + _require( + isinstance(contract, str) and isinstance(reference, str), + f"{manifest_relative}: plan contract and schema must be declared", + ) + return { + contract: _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + } + + references = manifest.get("plan_schemas") + _require(isinstance(references, dict), f"{manifest_relative}: plan_schemas must be an object") + schemas: dict[str, Mapping[str, object]] = {} + for contract, reference in references.items(): + _require( + isinstance(contract, str) and isinstance(reference, str), + f"{manifest_relative}: plan schema references must be strings", + ) + schemas[contract] = _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + return schemas + + +def _version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def _expand_frontend_cases( + version: str, + manifest: Mapping[str, object], + manifests: Mapping[str, Mapping[str, object]], +) -> tuple[Mapping[str, object], ...]: + """Return inherited cases with contract override followed by declared cases.""" + + def expand( + current_version: str, + current_manifest: Mapping[str, object], + stack: tuple[str, ...], + ) -> tuple[Mapping[str, object], ...]: + _require( + current_version not in stack, + f"{FRONTEND_MANIFESTS[current_version]}: inherited manifest cycle", + ) + current_stack = (*stack, current_version) + current_relative = FRONTEND_MANIFESTS[current_version] + declared_cases = current_manifest.get("cases") + _require(isinstance(declared_cases, list), f"{current_relative}: cases must be a list") + declared_by_id = { + case["id"]: case + for case in declared_cases + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + + entries = current_manifest.get("inherited_manifests", []) + _require( + isinstance(entries, list), + f"{current_relative}: inherited_manifests must be a list", + ) + expanded: list[Mapping[str, object]] = [] + for index, entry in enumerate(entries): + context = f"{current_relative}.inherited_manifests[{index}]" + _require(isinstance(entry, dict), f"{context}: must be an object") + reference = entry.get("manifest") + try: + parts = _canonical_parts(reference) + except ArtifactValidationError: + raise ArtifactValidationError( + f"{context}: invalid inherited manifest: {reference!r}" + ) from None + _require( + len(parts) == 1 and parts[0] == reference, + f"{context}: invalid inherited manifest: {reference!r}", + ) + target_versions = [ + target_version + for target_version, target_relative in FRONTEND_MANIFESTS.items() + if target_relative == f"conformance/{reference}" + and _version_key(target_version) < _version_key(current_version) + ] + _require( + len(target_versions) == 1 and target_versions[0] in manifests, + f"{context}: inherited manifest must target an older registered Frontend manifest", + ) + target_version = target_versions[0] + target_manifest = manifests[target_version] + _require( + entry.get("request_contract_override") == current_manifest.get("contract"), + f"{context}: request_contract_override must match current manifest contract", + ) + superseded = entry.get("superseded_cases") + _require(isinstance(superseded, dict), f"{context}: superseded_cases must be an object") + target_declared = target_manifest.get("cases") + _require( + isinstance(target_declared, list), + f"{FRONTEND_MANIFESTS[target_version]}: cases must be a list", + ) + target_by_id = { + case["id"]: case + for case in target_declared + if isinstance(case, dict) and isinstance(case.get("id"), str) + } + for old_id, replacement_id in superseded.items(): + _require( + isinstance(old_id, str) and isinstance(replacement_id, str), + f"{context}: superseded case IDs must be strings", + ) + _require( + old_id in target_by_id, + f"{context}: unknown inherited case: {old_id}", + ) + _require( + replacement_id in declared_by_id, + f"{context}: unknown replacement case: {replacement_id}", + ) + old_request = target_by_id[old_id].get("request") + replacement_request = declared_by_id[replacement_id].get("request") + _require( + isinstance(old_request, dict) and isinstance(replacement_request, dict), + f"{context}: superseded case request mismatch: {old_id}", + ) + expected_replacement_request = deepcopy(old_request) + expected_replacement_request["contract"] = entry["request_contract_override"] + _require( + replacement_request == expected_replacement_request, + f"{context}: superseded case request mismatch: {old_id}", + ) + + for inherited_case in expand(target_version, target_manifest, current_stack): + old_id = inherited_case["id"] + if old_id in superseded: + continue + case = deepcopy(inherited_case) + case["id"] = f"{target_version}/{old_id}" + request = case.get("request") + _require( + isinstance(request, dict), + f"{context}: inherited case request must be an object", + ) + request["contract"] = entry["request_contract_override"] + expanded.append(case) + + expanded.extend(deepcopy(case) for case in declared_cases) + seen: set[str] = set() + for case in expanded: + case_id = case.get("id") + _require( + isinstance(case_id, str) and case_id not in seen, + f"{current_relative}: duplicate expanded case id: {case_id}", + ) + seen.add(case_id) + return tuple(expanded) + + return expand(version, manifest, ()) + + +def _case_source_version(case_id: str, version: str) -> str: + if "/" in case_id: + source_version, _, _ = case_id.partition("/") + if source_version in FRONTEND_MANIFESTS: + return source_version + return version + + +def _resolve_frontend_plans( + root: Path, + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], + effective_cases: Mapping[str, tuple[Mapping[str, object], ...]], +) -> Mapping[str, Mapping[str, Mapping[str, object]]]: + """Return version -> case-id -> resolved canonical plan for successes.""" + resolved: dict[str, dict[str, Mapping[str, object]]] = {} + for version in ("0.1", "0.2", "0.3"): + manifest = manifests[version] + manifest_relative = FRONTEND_MANIFESTS[version] + request_reference = manifest.get("request_schema") + _require( + isinstance(request_reference, str), + f"{manifest_relative}: request_schema must be a string", + ) + request_schema = _schema( + root, + _resolve_manifest_reference(manifest_relative, request_reference), + ) + plan_schemas = _frontend_plan_schemas(root, version, manifest) + cases = effective_cases[version] + successful: dict[str, Mapping[str, object]] = {} + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + source_version = _case_source_version(case["id"], version) + expected_error = _expected_error(case, context) + request = case.get("request") + _require(isinstance(request, dict), f"{context}.request: must be an object") + source_text = request.get("source_text") + if isinstance(source_text, str): + actual_source_hash = source_hash(source_text) + _require( + isinstance(case.get("expected_source_hash"), str) + and case.get("expected_source_hash") == actual_source_hash, + f"{context}: source hash mismatch", + ) + request_valid = _schema_is_valid(request, request_schema) + _require( + request_valid, + f"{context}.request: schema validation failed", + ) + + plan_fields = ( + "expected_plan", + "expected_plan_0_1", + "expected_plan_case", + "expected_plan_case_0_1", + ) + present_plan_fields = [field for field in plan_fields if field in case] + if expected_error is not None and expected_error != "plan_schema_invalid": + for field in (*plan_fields, "expected_plan_contract", "expected_plan_hash"): + _require( + field not in case, + f"{context}: failing case must omit {field}", + ) + continue + + _require( + len(present_plan_fields) == 1, + f"{context}: successful case must declare exactly one expected plan", + ) + expected_field = present_plan_fields[0] + plan: Mapping[str, object] + referenced_plan_hash: object = None + if expected_field in {"expected_plan", "expected_plan_0_1"}: + inline_plan = case.get(expected_field) + _require( + isinstance(inline_plan, dict), + f"{context}.{expected_field}: must be an object", + ) + plan = inline_plan + elif expected_field == "expected_plan_case": + plan_version = "0.1" if source_version == "0.1" else "0.2" + expected_contract = case.get("expected_plan_contract") + _require( + expected_contract is None + or expected_contract == manifests[source_version].get( + "plan_contract", "openstatspec-transformation-plan-v0.2" + ), + f"{context}: expected_plan_contract does not match expected_plan_case", + ) + reference = case.get(expected_field) + _require( + isinstance(reference, str), + f"{context}.{expected_field}: must be a string", + ) + referenced_case = plan_cases[plan_version].get(reference) + _require( + referenced_case is not None, + f"{context}: unknown plan case: {reference}", + ) + plan = referenced_case["plan"] + referenced_plan_hash = referenced_case.get("expected_plan_hash") + else: + _require( + source_version in {"0.2", "0.3"}, + f"{context}.{expected_field}: only Frontend 0.2 and 0.3 may use this reference", + ) + reference = case.get(expected_field) + _require( + isinstance(reference, str), + f"{context}.{expected_field}: must be a string", + ) + frontend_01_plan = resolved["0.1"].get(reference) + _require( + frontend_01_plan is not None, + f"{context}: unknown Frontend 0.1 plan case: {reference}", + ) + plan = frontend_01_plan + + contract = plan.get("contract") + _require( + isinstance(contract, str) and contract in plan_schemas, + f"{context}: unsupported expected plan contract", + ) + try: + Draft202012Validator(plan_schemas[contract]).validate(plan) + except ValidationError as error: + if expected_error == "plan_schema_invalid": + _require( + case.get("expected_plan_hash") is None, + f"{context}: plan_schema_invalid must omit expected_plan_hash or set it to null", + ) + continue + raise ArtifactValidationError( + f"{context}: expected plan schema validation failed: {error.message}" + ) from None + _require( + expected_error != "plan_schema_invalid", + f"{context}: plan_schema_invalid requires a schema-invalid plan", + ) + declared_contract = case.get("expected_plan_contract") + _require( + declared_contract is None or declared_contract == contract, + f"{context}: expected_plan_contract does not match plan", + ) + expected_hash = case.get("expected_plan_hash", referenced_plan_hash) + actual_hash = _sha256(canonical_json_bytes(plan)) + _require( + isinstance(expected_hash, str) and expected_hash == actual_hash, + f"{context}: plan hash mismatch", + ) + successful[case["id"]] = plan + resolved[version] = successful + return resolved + + +def _validate_binding_manifests( + manifests: Mapping[str, Mapping[str, object]], + plan_cases: Mapping[str, Mapping[str, object]], + frontend_plans: Mapping[str, Mapping[str, Mapping[str, object]]], + frontend_manifests: Mapping[str, Mapping[str, object]], +) -> None: + for version, manifest in manifests.items(): + manifest_relative = BINDING_MANIFESTS[version] + cases = manifest.get("cases") + _require(isinstance(cases, list), f"{manifest_relative}: cases must be a list") + for index, case in enumerate(cases): + context = f"{manifest_relative}.cases[{index}]" + _require(isinstance(case, dict), f"{context}: each case must be an object") + expected_error = _expected_error(case, context) + if expected_error is not None: + _require( + case.get("mutation_started") is False + or ( + case.get("mutation_started") is True + and case.get("failure_point") == "after_data_and_metadata_before_audit" + ), + f"{context}: failing binding case must set mutation_started to false or declare failure_point=after_data_and_metadata_before_audit", + ) + if version != "0.2": + continue + + has_plan_reference = "applied_plan_case" in case + has_frontend_reference = "applied_frontend_case" in case + _require( + has_plan_reference == has_frontend_reference, + f"{context}: binding plan and frontend references must be paired", + ) + if not has_plan_reference: + continue + + plan_reference = case.get("applied_plan_case") + frontend_reference = case.get("applied_frontend_case") + _require( + isinstance(plan_reference, str), + f"{context}: applied_plan_case must be a string", + ) + _require( + isinstance(frontend_reference, str), + f"{context}: applied_frontend_case must be a string", + ) + plan_case = plan_cases["0.2"].get(plan_reference) + _require( + plan_case is not None, + f"{context}: unknown plan case: {plan_reference}", + ) + frontend_plan = frontend_plans["0.2"].get(frontend_reference) + _require( + frontend_plan is not None, + f"{context}: unknown frontend case: {frontend_reference}", + ) + _require( + frontend_plan == plan_case["plan"], + f"{context}: binding reference mismatch", + ) + + expected_audit = case.get("expected_audit") + if expected_audit is None: + continue + _require( + isinstance(expected_audit, dict), + f"{context}.expected_audit: must be an object", + ) + plan = plan_case["plan"] + frontend_case = next( + ( + frontend_case + for frontend_case in frontend_manifests["0.2"].get("cases", []) + if isinstance(frontend_case, dict) + and frontend_case.get("id") == frontend_reference + ), + None, + ) + _require( + isinstance(frontend_case, dict), + f"{context}: unknown frontend case: {frontend_reference}", + ) + expected_values = { + "plan_hash": plan_case.get("expected_plan_hash"), + "source_hash": frontend_case.get("expected_source_hash"), + "canonical_plan_json": canonical_json_bytes(plan).decode("utf-8"), + "operation_count": len(plan["operations"]), + } + for field, expected in expected_values.items(): + if field in expected_audit: + _require( + expected_audit[field] == expected, + f"{context}.expected_audit.{field}: audit identity mismatch", + ) + + +def _require_exact_field(value: object, expected: object, context: str) -> None: + _require(value == expected, f"{context}: has an unexpected value") + + +def validate_release_metadata(root: Path) -> None: + for relative in V030_PROFILE_DOCUMENTS: + text = " ".join(_load_text(root, relative).split()) + _require( + "Status: released in OpenStatSpec `v0.3.0`" in text, + f"{relative}: release status is inconsistent with v0.3.0", + ) + _require( + "release candidate for the planned OpenStatSpec `v0.3.0`" not in text, + f"{relative}: stale release status remains", + ) + roadmap = " ".join(_load_text(root, "ROADMAP.md").split()) + _require("`v0.3.0` is the" in roadmap, "ROADMAP release tag mismatch") + _require( + "current public specification release and is immutable" in roadmap, + "ROADMAP release status mismatch", + ) + _require(V030_COMMIT in roadmap, "ROADMAP v0.3.0 commit mismatch") + + frontend_03 = " ".join(_load_text(root, FRONTEND_03_PROFILE_DOCUMENT).split()) + _require( + "Status: release candidate for the next OpenStatSpec minor release." in frontend_03 + and "is not published stable until a protected or signed specification tag targets" in frontend_03 + and "its exact reviewed commit and tag-context CI passes." in frontend_03, + f"{FRONTEND_03_PROFILE_DOCUMENT}: release-candidate status is missing", + ) + for phrase in ( + "Pin the exact future specification release commit.", + "Accept only the exact Frontend 0.3 request contract for this suite.", + "Run all 90 effective Frontend 0.3 cases.", + "Preserve every inherited Plan 0.1/0.2 object and hash.", + "Pass all 35 declared cases with exact source/plan hashes and diagnostics.", + "Continue running existing Plan 0.1/0.2 and In-Place 0.1/0.2 suites.", + "Keep MySQL/MariaDB/Dolt pre-provisioning and Dolt caller-owned commit policy", + "Adapter evidence does not block specification publication.", + ): + _require( + phrase in frontend_03, + f"{FRONTEND_03_PROFILE_DOCUMENT}: adapter claim gate is missing {phrase}", + ) + + classification = " ".join( + _load_text(root, "docs/spss-frontend-roadmap.md").split() + ) + for phrase in ( + "Frontend 0.3, Plan 0.1/0.2", + "New Plan/Frontend/Binding generation", + "Separate case-transformation profile", + "Explicit non-goal", + "Milestone 0: integrity and classification\n\n- [x] Specification-complete", + "Milestone 1: Frontend 0.3 over Plan 0.1/0.2\n\n- [x] Specification-complete", + "Adapter claims remain pending and separately gated", + ): + _require( + " ".join(phrase.split()) in classification, + f"SPSS classification is missing {phrase}", + ) + + +def validate_contract_artifacts(root: Path) -> ArtifactInventory: + plan_manifests: dict[str, dict[str, object]] = {} + plan_contracts = { + "0.1": "openstatspec-transformation-plan-v0.1", + "0.2": "openstatspec-transformation-plan-v0.2", + } + for version, manifest_relative in PLAN_MANIFESTS.items(): + manifest = _load_manifest(root, manifest_relative, version, plan_contracts[version]) + schema_reference = { + "0.1": "../transformation/plan-0.1.schema.json", + "0.2": "../transformation/plan-0.2.schema.json", + }[version] + _require_exact_field( + manifest.get("schema"), + schema_reference, + f"{manifest_relative}: schema", + ) + schema_relative = _resolve_manifest_reference(manifest_relative, schema_reference) + _schema(root, schema_relative) + plan_manifests[version] = manifest + plan_cases = _validate_plan_manifests(root, plan_manifests) + + frontend_manifests: dict[str, dict[str, object]] = {} + frontend_contracts = { + "0.1": "openstatspec-spss-syntax-frontend-v0.1", + "0.2": "openstatspec-spss-syntax-frontend-v0.2", + "0.3": "openstatspec-spss-syntax-frontend-v0.3", + } + for version, manifest_relative in FRONTEND_MANIFESTS.items(): + request_reference = { + "0.1": "../transformation/spss-syntax-frontend-0.1.schema.json", + "0.2": "../transformation/spss-syntax-frontend-0.2.schema.json", + "0.3": "../transformation/spss-syntax-frontend-0.3.schema.json", + }[version] + manifest = _load_manifest(root, manifest_relative, version, frontend_contracts[version]) + _require_exact_field( + manifest.get("request_schema"), + request_reference, + f"{manifest_relative}: request_schema", + ) + request_relative = _resolve_manifest_reference(manifest_relative, request_reference) + _schema(root, request_relative) + + if version == "0.1": + _require_exact_field( + manifest.get("plan_contract"), + "openstatspec-transformation-plan-v0.1", + f"{manifest_relative}: plan_contract", + ) + plan_reference = "../transformation/plan-0.1.schema.json" + _require_exact_field( + manifest.get("plan_schema"), + plan_reference, + f"{manifest_relative}: plan_schema", + ) + resolved_plan_reference = _resolve_manifest_reference( + manifest_relative, + plan_reference, + ) + _schema(root, resolved_plan_reference) + else: + _require_exact_field( + manifest.get("plan_contracts"), + [ + "openstatspec-transformation-plan-v0.1", + "openstatspec-transformation-plan-v0.2", + ], + f"{manifest_relative}: plan_contracts", + ) + plan_references = { + "openstatspec-transformation-plan-v0.1": "../transformation/plan-0.1.schema.json", + "openstatspec-transformation-plan-v0.2": "../transformation/plan-0.2.schema.json", + } + _require_exact_field( + manifest.get("plan_schemas"), + plan_references, + f"{manifest_relative}: plan_schemas", + ) + for reference in plan_references.values(): + _schema( + root, + _resolve_manifest_reference(manifest_relative, reference), + ) + frontend_manifests[version] = manifest + frontend_effective_cases = { + version: _expand_frontend_cases(version, manifest, frontend_manifests) + for version, manifest in frontend_manifests.items() + } + frontend_plans = _resolve_frontend_plans( + root, + frontend_manifests, + plan_cases, + frontend_effective_cases, + ) + + binding_contracts = { + "0.1": "openstatspec-in-place-transformation-v0.1", + "0.2": "openstatspec-in-place-transformation-v0.2", + } + binding_manifests: dict[str, dict[str, object]] = {} + for version, manifest_relative in BINDING_MANIFESTS.items(): + manifest = _load_manifest(root, manifest_relative, version, binding_contracts[version]) + if version == "0.2": + audit_reference = "../sql/transformation-plan-profile-schema.sql" + _require_exact_field( + manifest.get("audit_schema"), + audit_reference, + f"{manifest_relative}: audit_schema", + ) + resolved_audit_reference = _resolve_manifest_reference( + manifest_relative, + audit_reference, + ) + _require_file(root, resolved_audit_reference) + binding_manifests[version] = manifest + _validate_binding_manifests( + binding_manifests, + plan_cases, + frontend_plans, + frontend_manifests, + ) + + return ArtifactInventory( + plan_cases={version: len(manifest["cases"]) for version, manifest in plan_manifests.items()}, + frontend_declared_cases={ + version: len(manifest["cases"]) + for version, manifest in frontend_manifests.items() + }, + frontend_effective_cases={ + version: len(cases) + for version, cases in frontend_effective_cases.items() + }, + binding_cases={ + version: len(manifest["cases"]) + for version, manifest in binding_manifests.items() + }, + ) diff --git a/tests/test_contract_artifacts.py b/tests/test_contract_artifacts.py new file mode 100644 index 0000000..eb5f2ce --- /dev/null +++ b/tests/test_contract_artifacts.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from pathlib import Path + +import pytest + +from openstatspec_specification.artifacts import ArtifactValidationError +from openstatspec_specification.artifacts import canonical_json_bytes +from openstatspec_specification.artifacts import validate_release_metadata +from openstatspec_specification.artifacts import validate_contract_artifacts +from openstatspec_specification.artifacts import source_hash + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_frontend_03_final_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} + + +def test_released_contract_artifact_inventory_is_valid() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.plan_cases == {"0.1": 4, "0.2": 26} + assert inventory.frontend_declared_cases == {"0.1": 13, "0.2": 44, "0.3": 35} + assert inventory.frontend_effective_cases == {"0.1": 13, "0.2": 44, "0.3": 90} + assert inventory.binding_cases == {"0.1": 6, "0.2": 11} + + +def test_frontend_03_request_schema_is_registered() -> None: + inventory = validate_contract_artifacts(ROOT) + assert "0.3" in inventory.frontend_declared_cases + + +def test_frontend_03_predicate_and_open_range_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 + + +def test_frontend_03_schema_changes_only_contract_identity() -> None: + old = json.loads((ROOT / "transformation/spss-syntax-frontend-0.2.schema.json").read_text()) + new = json.loads((ROOT / "transformation/spss-syntax-frontend-0.3.schema.json").read_text()) + assert new["$id"].endswith("spss-syntax-frontend-0.3.schema.json") + assert new["title"] == "OpenStatSpec SPSS-like Syntax Frontend Request 0.3" + assert new["properties"]["contract"]["const"] == "openstatspec-spss-syntax-frontend-v0.3" + for document in (old, new): + document.pop("$id") + document.pop("title") + document["properties"]["contract"].pop("const") + assert new == old + + +def test_frontend_03_inherits_released_cases_under_new_request_contract() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 + + +def test_frontend_03_comment_and_varlist_case_inventory() -> None: + inventory = validate_contract_artifacts(ROOT) + assert inventory.frontend_declared_cases["0.3"] == 35 + assert inventory.frontend_effective_cases["0.3"] == 90 + + +def test_value_labels_then_add_value_labels_keeps_both_ordered_replacements() -> None: + manifest = json.loads( + (ROOT / "conformance/spss-syntax-frontend-0.3.json").read_text(encoding="utf-8") + ) + case = next( + case + for case in manifest["cases"] + if case["id"] == "value-labels-then-add-value-labels-uses-ordered-state" + ) + plan = case["expected_plan_0_1"] + assert [operation["op"] for operation in plan["operations"]] == [ + "replace_value_labels", + "replace_value_labels", + ] + assert plan["operations"][0]["labels"][0]["label"] == "Five" + assert plan["operations"][1]["labels"][-1]["label"] == "Two" + assert case["expected_source_hash"] == source_hash(case["request"]["source_text"]) + assert case["expected_plan_hash"] == hashlib.sha256( + canonical_json_bytes(plan) + ).hexdigest() + + +def test_frontend_03_release_candidate_documentation_is_complete() -> None: + readme = (ROOT / "README.md").read_text(encoding="utf-8") + changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") + example = (ROOT / "examples/spss-syntax-transformation-0.3.md").read_text(encoding="utf-8") + assert "spss-syntax-frontend-0.3.schema.json" in readme + assert "SPSS Syntax Frontend 0.3" in changelog + assert "Frontend 0.3" in roadmap + assert "ADD VALUE LABELS" in example + assert "Plan 0.1/0.2" in example + + +def test_inherited_manifest_path_escape_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["inherited_manifests"][0]["manifest"] = "../outside.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="inherited manifest"): + validate_contract_artifacts(root) + + +@pytest.mark.parametrize("artifact_type", ["directory", "fifo", "symlink"]) +def test_existing_frontend_03_manifest_artifact_type_fails_closed( + tmp_path: Path, artifact_type: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + path.unlink() + if artifact_type == "directory": + path.mkdir() + elif artifact_type == "fifo": + os.mkfifo(path) + else: + path.symlink_to(root / "conformance") + with pytest.raises(ArtifactValidationError, match="artifact"): + validate_contract_artifacts(root) + + +def test_v030_profile_status_is_consistently_released() -> None: + validate_release_metadata(ROOT) + + +def test_release_candidate_wording_fails_for_published_profile(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + text = path.read_text(encoding="utf-8").replace( + "Status: released in OpenStatSpec `v0.3.0`", + "Status: release candidate for OpenStatSpec `v0.3.0`", + ) + path.write_text(text, encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="release status"): + validate_release_metadata(root) + + +def test_release_document_symlink_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "docs/spss-syntax-frontend-profile-0.2.md" + path.unlink() + path.symlink_to(root / "docs/transformation-plan-profile-0.2.md") + with pytest.raises(ArtifactValidationError, match="artifact path traverses symlink"): + validate_release_metadata(root) + + +def copied_artifacts(tmp_path: Path) -> Path: + for directory in ("conformance", "transformation", "sql", "docs"): + shutil.copytree(ROOT / directory, tmp_path / directory) + shutil.copy2(ROOT / "CHANGELOG.md", tmp_path / "CHANGELOG.md") + shutil.copy2(ROOT / "ROADMAP.md", tmp_path / "ROADMAP.md") + return tmp_path + + +def test_duplicate_manifest_case_id_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"].append(manifest["cases"][0]) + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="duplicate case id"): + validate_contract_artifacts(root) + + +def test_invalid_json_schema_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "transformation/plan-0.1.schema.json" + schema = json.loads(path.read_text(encoding="utf-8")) + schema["type"] = "not-a-json-schema-type" + path.write_text(json.dumps(schema), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="invalid JSON Schema"): + validate_contract_artifacts(root) + + +@pytest.mark.parametrize( + ("manifest_name", "field"), + [ + ("transformation-plan-0.2.json", "expected_plan_hash"), + ("spss-syntax-frontend-0.2.json", "expected_source_hash"), + ], +) +def test_hash_mismatch_fails_closed( + tmp_path: Path, manifest_name: str, field: str, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance" / manifest_name + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0][field] = "0" * 64 + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="hash mismatch"): + validate_contract_artifacts(root) + + +def test_missing_frontend_plan_reference_fails_closed(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["expected_plan_case"] = "absent-plan" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="unknown plan case"): + validate_contract_artifacts(root) + + +def test_binding_plan_and_frontend_references_must_agree(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/in-place-transformation-0.2.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + manifest["cases"][0]["applied_plan_case"] = "strict-greater-than" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="binding reference mismatch"): + validate_contract_artifacts(root) + + +def test_semantic_plan_failure_must_remain_schema_valid(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "reject-descending-range") + case["plan"]["operations"][0]["unexpected"] = True + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="semantic failure must have a schema-valid plan"): + validate_contract_artifacts(root) + + +def test_plan_schema_invalid_failure_requires_invalid_plan(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/transformation-plan-0.1.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "reject-descending-range") + case["expected_error"] = "plan_schema_invalid" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="plan_schema_invalid requires a schema-invalid plan"): + validate_contract_artifacts(root) + + +def test_frontend_plan_schema_invalid_case_requires_valid_request_and_invalid_plan( + tmp_path: Path, +) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + case = next(case for case in manifest["cases"] if case["id"] == "formats-grouped-to-expands-source-order") + case["expected_error"] = "plan_schema_invalid" + case["expected_plan"]["operations"][0]["unexpected"] = True + case.pop("expected_plan_contract") + case.pop("expected_plan_hash") + path.write_text(json.dumps(manifest), encoding="utf-8") + validate_contract_artifacts(root) + + +def test_superseded_replacement_must_keep_request_identity(tmp_path: Path) -> None: + root = copied_artifacts(tmp_path) + path = root / "conformance/spss-syntax-frontend-0.3.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + replacement = next( + case for case in manifest["cases"] if case["id"] == "inline-block-comment-is-ignored" + ) + replacement["request"]["input_alias"] = "other" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ArtifactValidationError, match="superseded case request mismatch"): + validate_contract_artifacts(root) diff --git a/tools/validate_repository.py b/tools/validate_repository.py index e8808b6..9f9e7cc 100644 --- a/tools/validate_repository.py +++ b/tools/validate_repository.py @@ -13,6 +13,11 @@ DoltDeclarationError, main, ) +from openstatspec_specification.artifacts import ( # noqa: E402 + ArtifactValidationError, + validate_contract_artifacts, + validate_release_metadata, +) def require(condition: bool, message: str) -> None: @@ -136,10 +141,19 @@ def validate_repository_controls() -> None: ): require(f"CREATE TABLE {table} (" in schema, f"Schema table is missing: {table}") + validate_release_metadata(ROOT) + if __name__ == "__main__": try: main() + inventory = validate_contract_artifacts(ROOT) validate_repository_controls() - except DoltDeclarationError as error: + print( + "Validated contract artifacts: " + f"Plan {dict(inventory.plan_cases)}, " + f"Frontend {dict(inventory.frontend_effective_cases)}, " + f"Binding {dict(inventory.binding_cases)}." + ) + except (DoltDeclarationError, ArtifactValidationError) as error: raise SystemExit(str(error)) from error diff --git a/transformation/spss-syntax-frontend-0.3.schema.json b/transformation/spss-syntax-frontend-0.3.schema.json new file mode 100644 index 0000000..5523006 --- /dev/null +++ b/transformation/spss-syntax-frontend-0.3.schema.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openstatspec.org/schemas/spss-syntax-frontend-0.3.schema.json", + "title": "OpenStatSpec SPSS-like Syntax Frontend Request 0.3", + "type": "object", + "additionalProperties": false, + "required": ["contract", "input_alias", "input_schema", "source_text"], + "properties": { + "contract": {"const": "openstatspec-spss-syntax-frontend-v0.3"}, + "input_alias": {"type": "string", "minLength": 1}, + "input_schema": { + "type": "object", "additionalProperties": false, "required": ["variables"], + "properties": { + "variables": { + "type": "array", "minItems": 1, + "items": { + "type": "object", "additionalProperties": false, "required": ["name", "storage_kind"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "storage_kind": {"enum": ["numeric", "string"]}, + "variable_label": {"type": ["string", "null"]}, + "value_labels": {"type": "array", "items": {"$ref": "#/$defs/value_label"}}, + "format_family": {"type": ["string", "null"]}, + "width": {"type": ["integer", "null"], "minimum": 1}, + "decimals": {"type": ["integer", "null"], "minimum": 0}, + "measurement_level": {"enum": ["nominal", "ordinal", "scale", null]} + } + } + } + } + }, + "source_text": {"type": "string", "minLength": 1} + }, + "$defs": { + "binary64_value": { + "type": "object", "additionalProperties": false, "required": ["type", "bits"], + "properties": { + "type": {"const": "binary64"}, + "bits": {"type": "string", "pattern": "^(?!8000000000000000$)(?!(?:7ff|fff)[0-9a-f]{13}$)[0-9a-f]{16}$"} + } + }, + "string_value": { + "type": "object", "additionalProperties": false, "required": ["type", "value"], + "properties": {"type": {"const": "string"}, "value": {"type": "string"}} + }, + "typed_value": {"oneOf": [{"$ref": "#/$defs/binary64_value"}, {"$ref": "#/$defs/string_value"}]}, + "value_label": { + "type": "object", + "additionalProperties": false, + "required": ["value", "label"], + "properties": { + "value": {"$ref": "#/$defs/typed_value"}, + "label": {"type": "string"} + } + } + } +}