From d29b65940dad4161880c2e1ab853d6e97416f06d Mon Sep 17 00:00:00 2001 From: Arcangelo Massari Date: Sun, 31 May 2026 12:32:10 +0200 Subject: [PATCH 1/2] ci(shacl): add shape validation for sample data --- .github/scripts/validate_shacl.py | 61 +++++++++++ .../workflows/validate_specs_and_jsons.yml | 19 ++++ .gitmodules | 3 + data-model | 1 + pyproject.toml | 5 + uv.lock | 102 +++++++++++++++++- 6 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/validate_shacl.py create mode 100644 .gitmodules create mode 160000 data-model diff --git a/.github/scripts/validate_shacl.py b/.github/scripts/validate_shacl.py new file mode 100644 index 0000000..e3aca07 --- /dev/null +++ b/.github/scripts/validate_shacl.py @@ -0,0 +1,61 @@ +import json +import sys +from pathlib import Path + +from pyshacl import validate +from rdflib import Graph + +OPENAPI_VER_DIR = Path("openapi/ver") +SHACL_DIR = Path("data-model/shacl") + + +def load_shapes(version): + shapes_path = SHACL_DIR / version / "shacl.ttl" + if not shapes_path.exists(): + return None + g = Graph() + g.parse(str(shapes_path), format="turtle") + return g + + +def validate_file(json_path, shapes_graph): + with open(json_path, encoding="utf-8") as f: + data = json.load(f) + data_graph = Graph() + data_graph.parse(data=json.dumps(data), format="json-ld") + if len(data_graph) == 0: + return False, "Parsed RDF graph is empty (context resolution may have failed)" + conforms, _, results_text = validate( + data_graph=data_graph, shacl_graph=shapes_graph, debug=False + ) + return conforms, results_text + + +def main(): + failures = [] + versions_found = 0 + + for version_dir in sorted(OPENAPI_VER_DIR.iterdir()): + sample_data_dir = version_dir / "sample_data" + json_files = sorted(sample_data_dir.glob("**/*.json")) if sample_data_dir.is_dir() else [] + if not json_files: + continue + + shapes_graph = load_shapes(version_dir.name) + + versions_found += 1 + for json_path in json_files: + conforms, results_text = validate_file(json_path, shapes_graph) + if conforms: + print(f"PASS {json_path}") + else: + print(f"FAIL {json_path}\n{results_text}") + failures.append(json_path) + + print(f"\n{versions_found} versions validated, {len(failures)} failures") + if failures: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/validate_specs_and_jsons.yml b/.github/workflows/validate_specs_and_jsons.yml index 8fe5df2..dba2abf 100644 --- a/.github/workflows/validate_specs_and_jsons.yml +++ b/.github/workflows/validate_specs_and_jsons.yml @@ -5,10 +5,12 @@ on: paths: - 'openapi/ver/**/*.json' - 'openapi/ver/**/skg-if-openapi.yaml' + - 'data-model/shacl/**' pull_request: paths: - 'openapi/ver/**/*.json' - 'openapi/ver/**/skg-if-openapi.yaml' + - 'data-model/shacl/**' jobs: validate_yaml_files: @@ -26,6 +28,23 @@ jobs: - if: ${{ failure() }} run: exit() + validate_shacl: + runs-on: ubuntu-latest + name: Validate SHACL Shapes + needs: validate_yaml_files + + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + submodules: true + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8 + + - name: Validate sample data against SHACL shapes + run: uv run python .github/scripts/validate_shacl.py + validate_json_files: runs-on: ubuntu-latest name: Validate JSON Files diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..431cadb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "data-model"] + path = data-model + url = https://github.com/skg-if/data-model diff --git a/data-model b/data-model new file mode 160000 index 0000000..2d98aef --- /dev/null +++ b/data-model @@ -0,0 +1 @@ +Subproject commit 2d98aef4c452d71641baeb73ad9bfd429a4d915d diff --git a/pyproject.toml b/pyproject.toml index dc63df6..4aebde6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,3 +19,8 @@ dependencies = [ [tool.uv.sources] plain-text-markdown-extention = { git = "https://github.com/kostyachum/python-markdown-plain-text.git" } + +[dependency-groups] +dev = [ + "pyshacl>=0.31.0", +] diff --git a/uv.lock b/uv.lock index b3db614..2dd3844 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" [[package]] @@ -133,6 +133,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "html5rdf" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/55/1b839c43f5ed8207e17a9a02d8b395179520b8b4f00c00a41e113bc205ca/html5rdf-1.2.1.tar.gz", hash = "sha256:ace9b420ce52995bb4f05e7425eedf19e433c981dfe7a831ab391e2fa2e1a195", size = 287899, upload-time = "2024-10-30T05:06:56.384Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/c9/f6e1e8567660bc5b0aba281f2b0017b2a7665fcad6bf3ed67286a0c72cd4/html5rdf-1.2.1-py2.py3-none-any.whl", hash = "sha256:1f519121bc366af3e485310dc8041d2e86e5173c1a320fac3dc9d2604069b83e", size = 109765, upload-time = "2024-10-30T05:06:52.507Z" }, +] + [[package]] name = "httpcore" version = "1.0.9" @@ -179,11 +188,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/3f/afe76f8e2246ffbc867440cbcf90525264df0e658f8a5ca1f872b3f6192a/markdown-3.8-py3-none-any.whl", hash = "sha256:794a929b79c5af141ef5ab0f2f642d0f7b1872981250230e72682346f7cc90dc", size = 106210, upload-time = "2025-04-11T14:42:49.178Z" }, ] +[[package]] +name = "owlrl" +version = "7.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rdflib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/fc/ce12482d096d65fff01af58f555a6f25e9dbf416fad5d99f91eaab0e11ca/owlrl-7.1.4.tar.gz", hash = "sha256:60bd4067e346b9111f0a2924565afe97ac6595b98b2bbe953928b5113971daf7", size = 44420, upload-time = "2025-07-29T00:17:27.608Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/78/f857ff1a7207e967dc5e8414bbcc15e0aa5cf45f693b1d2ebe2afb3eb1ce/owlrl-7.1.4-py3-none-any.whl", hash = "sha256:e78b46020169783345636da93a467d318f18700c483184dd15e885850cf64775", size = 51981, upload-time = "2025-07-29T00:17:26.229Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + [[package]] name = "plain-text-markdown-extention" version = "1.0.0" source = { git = "https://github.com/kostyachum/python-markdown-plain-text.git#8740e7ee364ac8ae9c436dcff58f8bb862ce93e5" } +[[package]] +name = "prettytable" +version = "3.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/45/b0847d88d6cfeb4413566738c8bbf1e1995fad3d42515327ff32cc1eb578/prettytable-3.17.0.tar.gz", hash = "sha256:59f2590776527f3c9e8cf9fe7b66dd215837cca96a9c39567414cbc632e8ddb0", size = 67892, upload-time = "2025-11-14T17:33:20.212Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/8c/83087ebc47ab0396ce092363001fa37c17153119ee282700c0713a195853/prettytable-3.17.0-py3-none-any.whl", hash = "sha256:aad69b294ddbe3e1f95ef8886a060ed1666a0b83018bbf56295f6f226c43d287", size = 34433, upload-time = "2025-11-14T17:33:19.093Z" }, +] + [[package]] name = "pydantic" version = "2.11.5" @@ -241,6 +283,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyshacl" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "owlrl" }, + { name = "packaging" }, + { name = "prettytable" }, + { name = "rdflib", extra = ["html"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/2d/8eaada41b9b57c028a54494688e45cfeefd6756098a6bf1bfa2dd9470cdf/pyshacl-0.31.0.tar.gz", hash = "sha256:327950875a5bb0d1a15c246a8a272b2dbf6bc9b96e28cfa8fdbfa4d73aadc0ba", size = 1406151, upload-time = "2026-01-16T06:34:06.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/3b/ebd7c9595fcdf176555aaf2fd2254f4d890658334ca3556b611e579f8294/pyshacl-0.31.0-py3-none-any.whl", hash = "sha256:5cae2184401d956b67deebb00e3c78ab7052784741a730e52e309e33c8a0b9a5", size = 1297210, upload-time = "2026-01-16T06:34:03.679Z" }, +] + [[package]] name = "python-dotenv" version = "1.1.0" @@ -250,6 +316,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/18/98a99ad95133c6a6e2005fe89faedf294a748bd5dc803008059409ac9b1e/python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d", size = 20256, upload-time = "2025-03-25T10:14:55.034Z" }, ] +[[package]] +name = "rdflib" +version = "7.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/f5/18bb77b7af9526add0c727a3b2048959847dc5fb030913e2918bf384fec3/rdflib-7.6.0.tar.gz", hash = "sha256:6c831288d5e4a5a7ece85d0ccde9877d512a3d0f02d7c06455d00d6d0ea379df", size = 4943826, upload-time = "2026-02-13T07:15:55.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c2/6604a71269e0c1bd75656d5a001432d16f2cc5b8c057140ec797155c295e/rdflib-7.6.0-py3-none-any.whl", hash = "sha256:30c0a3ebf4c0e09215f066be7246794b6492e054e782d7ac2a34c9f70a15e0dd", size = 615416, upload-time = "2026-02-13T07:15:46.487Z" }, +] + +[package.optional-dependencies] +html = [ + { name = "html5rdf" }, +] + [[package]] name = "requests" version = "2.32.4" @@ -307,6 +390,11 @@ dependencies = [ { name = "uvicorn" }, ] +[package.dev-dependencies] +dev = [ + { name = "pyshacl" }, +] + [package.metadata] requires-dist = [ { name = "dicttoxml", specifier = ">=1.7.16" }, @@ -321,6 +409,9 @@ requires-dist = [ { name = "uvicorn", specifier = ">=0.34.0" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pyshacl", specifier = ">=0.31.0" }] + [[package]] name = "sniffio" version = "1.3.1" @@ -396,3 +487,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/de/ad/713be230bcda622ea wheels = [ { url = "https://files.pythonhosted.org/packages/6d/0d/8adfeaa62945f90d19ddc461c55f4a50c258af7662d34b6a3d5d1f8646f6/uvicorn-0.34.3-py3-none-any.whl", hash = "sha256:16246631db62bdfbf069b0645177d6e8a77ba950cfedbfd093acef9444e4d885", size = 62431, upload-time = "2025-06-01T07:48:15.664Z" }, ] + +[[package]] +name = "wcwidth" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, +] From b8ce125e3258c4e9af9f0060e65786664b40ceab Mon Sep 17 00:00:00 2001 From: Arcangelo Massari Date: Tue, 16 Jun 2026 12:21:24 +0200 Subject: [PATCH 2/2] ci(shacl): trigger validation on submodule gitlink change --- .github/workflows/validate_specs_and_jsons.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate_specs_and_jsons.yml b/.github/workflows/validate_specs_and_jsons.yml index dba2abf..77c80c8 100644 --- a/.github/workflows/validate_specs_and_jsons.yml +++ b/.github/workflows/validate_specs_and_jsons.yml @@ -5,12 +5,12 @@ on: paths: - 'openapi/ver/**/*.json' - 'openapi/ver/**/skg-if-openapi.yaml' - - 'data-model/shacl/**' + - 'data-model' pull_request: paths: - 'openapi/ver/**/*.json' - 'openapi/ver/**/skg-if-openapi.yaml' - - 'data-model/shacl/**' + - 'data-model' jobs: validate_yaml_files: