From bd06574c89a540b9fb9ee5f21cd405d1e2e6d926 Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:29:35 +0300 Subject: [PATCH 1/8] release: prepare package metadata for v1.0.0 --- pyproject.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 32887e2..06ce12c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "turkishevalkit" -version = "0.13.0" +version = "1.0.0" description = "Human-in-the-loop Turkish AI evaluation workbench with portable datasets, rebuildable metadata indexing, reliability, review queues, calibration, and audio QA." readme = "README.md" requires-python = ">=3.11" @@ -21,7 +21,7 @@ keywords = [ "inter-rater-reliability", ] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", @@ -31,6 +31,12 @@ classifiers = [ ] dependencies = [] +[project.urls] +Homepage = "https://github.com/BLCCoreStudio/TurkishEvalKit" +Repository = "https://github.com/BLCCoreStudio/TurkishEvalKit" +Issues = "https://github.com/BLCCoreStudio/TurkishEvalKit/issues" +Changelog = "https://github.com/BLCCoreStudio/TurkishEvalKit/blob/main/CHANGELOG.md" + [project.optional-dependencies] workbench = [ "flask>=3.1,<4", From 38ff24922147596501373acf7a0a40568a2bd8c8 Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:29:48 +0300 Subject: [PATCH 2/8] docs: add v1 changelog --- CHANGELOG.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8bbc44d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog + +All notable TurkishEvalKit changes are documented here. The project follows semantic versioning from the v1.0 release onward. + +## 1.0.0 — 2026-09-16 + +### Evaluation and evidence +- Versioned Turkish text and audio rubrics with strict completeness validation and deterministic aggregate scores. +- Pairwise A/Tie/B evaluation with explicit overall preference and strength. +- Timestamped audio evidence with category, severity and human-authored notes. +- JSON, JSONL and CLI interfaces over the same typed domain model. + +### Review and revision workflows +- Local evaluator sessions and independent review, request-changes, escalation and adjudication flows. +- Immutable evaluation artifacts, append-only workflow events and explicit revision lineage. +- Action-oriented review queues derived from trusted history instead of a second mutable state store. + +### Calibration and reliability +- Multi-evaluator calibration for scalar, pairwise and timestamped-audio evaluations. +- Evidence-level disagreement exploration without turning agreement metrics into evaluator rankings. +- Repeated-task reliability analysis with applicability-aware Krippendorff alpha, Fleiss kappa and ICC(A,1). +- Local reliability workspace backed by the same core calculations used by the CLI. + +### Data portability and local operation +- Versioned evaluation-dataset interchange with JSON/JSONL conversion, dry-run import and exact-content deduplication. +- Optional rebuildable SQLite metadata index; canonical JSON artifacts remain the source of truth. +- Localhost workbench, review queue, calibration dashboard and reliability workspace. +- No required external AI service, telemetry, account or paid backend. + +### Quality and governance +- Python 3.11, 3.12 and 3.13 CI coverage. +- Ruff, strict mypy and pytest coverage gate of at least 90%. +- Wheel package-data smoke tests, CLI smoke tests, real localhost HTTP flows and Chromium browser E2E coverage. +- CodeQL, maintainer ownership, security/support policies and structured contribution templates. + +### v1 trust boundary +TurkishEvalKit standardizes human evaluation evidence and workflow mechanics. It does not claim that an evaluator is automatically correct, does not infer correctness from agreement alone, and does not silently import external workflow state as trusted local history. + +## 0.13.0 — 2026-09-03 +- Added the localhost Reliability Workspace and shared reliability navigation. + +## 0.12.0 — 2026-09-03 +- Added the optional rebuildable SQLite metadata index with canonical fallback. + +## 0.11.0 — 2026-09-03 +- Added versioned evaluation-dataset interchange and workspace import/export. + +## 0.10.0 and earlier +- Established repeated-task reliability, disagreement exploration, review queues, revision lineage, calibration, timestamped audio QA, pairwise evaluation and the localhost workbench. From 0dd0b7fcfd18841584fbbb15eeae6cf838825ae0 Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:30:07 +0300 Subject: [PATCH 3/8] ci: add reproducible GitHub release workflow --- .github/workflows/release.yml | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..06404d7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,76 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + cache: pip + + - name: Install build tooling + run: python -m pip install --disable-pip-version-check build + + - name: Verify tag matches package version + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python - <<'PY' + import os + import tomllib + from pathlib import Path + + tag = os.environ["RELEASE_TAG"] + if not tag.startswith("v"): + raise SystemExit(f"release tag must start with v: {tag}") + expected = tag[1:] + metadata = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + actual = metadata["project"]["version"] + if actual != expected: + raise SystemExit( + f"tag/package version mismatch: tag={expected!r}, package={actual!r}" + ) + PY + + - name: Build wheel and source distribution + run: python -m build + + - name: Verify built wheel in an isolated environment + run: | + python -m venv "$RUNNER_TEMP/release-venv" + "$RUNNER_TEMP/release-venv/bin/python" -m pip install --disable-pip-version-check dist/*.whl + "$RUNNER_TEMP/release-venv/bin/turkisheval" rubrics + "$RUNNER_TEMP/release-venv/bin/turkisheval" evaluate examples/text-evaluation.json --json \ + > "$RUNNER_TEMP/release-smoke.json" + grep -q '"normalized_score"' "$RUNNER_TEMP/release-smoke.json" + + - name: Write SHA-256 checksums + run: sha256sum dist/* > SHA256SUMS.txt + + - name: Publish GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ github.ref_name }}" \ + dist/* SHA256SUMS.txt \ + --repo "${{ github.repository }}" \ + --verify-tag \ + --title "TurkishEvalKit ${{ github.ref_name }}" \ + --generate-notes From b6e5b8f95525666a3ec19610dc218eab2f6eaaed Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:31:04 +0300 Subject: [PATCH 4/8] docs: graduate public status to v1 --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aa13087..25dcb94 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ TurkishEvalKit records native-language human judgments against explicit, versioned rubrics and turns them into inspectable local artifacts. It is designed for evaluator workflows, QA, research prototypes, and teams that need structured evidence without pretending an automated heuristic can replace the evaluator. -> **Status:** alpha (`0.13.x`). The project includes deterministic text/audio/pairwise evaluation, timestamped audio evidence, review/request-changes/adjudication workflows, immutable revision lineage, action-oriented review queues, multi-evaluator calibration, disagreement drill-down, repeated-task reliability statistics, a localhost reliability workspace, versioned evaluation-dataset interchange, optional rebuildable metadata indexing, JSON/JSONL/CLI interfaces, and localhost-only browser tools. +> **Status:** stable v1 (`1.0.x`). The v1 contract includes deterministic text/audio/pairwise evaluation, timestamped audio evidence, review/request-changes/adjudication workflows, immutable revision lineage, action-oriented review queues, multi-evaluator calibration, disagreement drill-down, repeated-task reliability statistics, a localhost reliability workspace, versioned evaluation-dataset interchange, optional rebuildable metadata indexing, JSON/JSONL/CLI interfaces, and localhost-only browser tools. See [`CHANGELOG.md`](CHANGELOG.md) for release history. ## Why this exists @@ -142,9 +142,9 @@ A reliability specification must declare `minimum_task_count` and the value must Reliability coefficients are not evaluator correctness scores and are never converted automatically into pass/fail thresholds or rankings. -#### Reliability Workspace (`0.13.x`) +#### Reliability Workspace (`1.0.x`) -The localhost workbench now exposes `/reliability`. +The localhost workbench exposes `/reliability`. - Candidate task units are derived from saved evaluation JSON plus trusted workflow-sidecar attribution. - Same-stimulus grouping uses task ID, evaluation type, rubric ID/version, and exact source stimulus. @@ -440,7 +440,7 @@ Population reliability consumes evaluation submissions but does not rewrite eval ## Non-goals -TurkishEvalKit does **not** currently: +TurkishEvalKit does **not**: - automatically decide whether an answer or voice sample is good; - send evaluation content to an external AI service; @@ -515,6 +515,7 @@ src/turkishevalkit/ ## Documentation +- [`CHANGELOG.md`](CHANGELOG.md) - [`docs/RUBRICS.md`](docs/RUBRICS.md) - [`docs/AUDIO_ANNOTATIONS.md`](docs/AUDIO_ANNOTATIONS.md) - [`docs/REVIEW_WORKFLOW.md`](docs/REVIEW_WORKFLOW.md) @@ -529,9 +530,9 @@ src/turkishevalkit/ - [`docs/WORKBENCH.md`](docs/WORKBENCH.md) - [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) -## Roadmap +## Post-v1 roadmap -Near-term work remains ordered around evaluator correctness rather than surface area: +Further work remains ordered around evaluator correctness rather than surface area: 1. explicit branching semantics only if real collaborative revision use cases justify the complexity; 2. shared audio-alignment primitives if additional evidence consumers need timestamp matching beyond calibration/explorer paths. From bcbfc951a95c8a7fb90a13ee34f2f628cb5fc8dd Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:39:07 +0300 Subject: [PATCH 5/8] release: align package runtime version with v1 --- src/turkishevalkit/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/turkishevalkit/__init__.py b/src/turkishevalkit/__init__.py index f21a448..834e4b1 100644 --- a/src/turkishevalkit/__init__.py +++ b/src/turkishevalkit/__init__.py @@ -161,4 +161,4 @@ "write_interchange_records", ] -__version__ = "0.13.0" +__version__ = "1.0.0" From 595a839206dfe50a6fd246eefea40150a5205591 Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:40:08 +0300 Subject: [PATCH 6/8] ci: align v1 version gates --- .../workflows/one-shot-v1-version-gates.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/one-shot-v1-version-gates.yml diff --git a/.github/workflows/one-shot-v1-version-gates.yml b/.github/workflows/one-shot-v1-version-gates.yml new file mode 100644 index 0000000..d2b9105 --- /dev/null +++ b/.github/workflows/one-shot-v1-version-gates.yml @@ -0,0 +1,56 @@ +name: One-shot v1 version gate refresh + +on: + push: + branches: + - release/v1-readiness + paths: + - .github/workflows/one-shot-v1-version-gates.yml + +permissions: + contents: write + +jobs: + refresh-version-gates: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout release branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: release/v1-readiness + fetch-depth: 0 + + - name: Align explicit v1 version assertions + run: | + python - <<'PY' + from pathlib import Path + + old = '__version__ == "0.13.0"' + new = '__version__ == "1.0.0"' + changed = [] + for path in Path('.github/workflows').glob('*.yml'): + if path.name == 'one-shot-v1-version-gates.yml': + continue + text = path.read_text(encoding='utf-8') + updated = text.replace(old, new) + if updated != text: + path.write_text(updated, encoding='utf-8') + changed.append(str(path)) + if not changed: + raise SystemExit('no pre-v1 version assertions found') + print('\n'.join(changed)) + PY + + - name: Verify no stale pre-v1 assertion remains + run: | + ! grep -R --line-number '__version__ == "0.13.0"' .github/workflows --include='*.yml' + + - name: Commit refreshed gates and remove this one-shot workflow + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/workflows + git rm .github/workflows/one-shot-v1-version-gates.yml + git commit -m "ci: align workflow version assertions with v1" + git push origin HEAD:release/v1-readiness From bd884c292f2068dbcc6b1f037e314ff5a8cf16cb Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:42:15 +0300 Subject: [PATCH 7/8] ci: align reliability workspace version with v1 --- .github/workflows/reliability-workspace.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reliability-workspace.yml b/.github/workflows/reliability-workspace.yml index abf1a09..1bfeb40 100644 --- a/.github/workflows/reliability-workspace.yml +++ b/.github/workflows/reliability-workspace.yml @@ -169,7 +169,7 @@ jobs: import turkishevalkit assert turkishevalkit.__version__ == version("turkishevalkit") - assert turkishevalkit.__version__ == "0.13.0" + assert turkishevalkit.__version__ == "1.0.0" PY reliability-browser-e2e: From e5328ff4242933a9dce438af2d787faac2f78e4a Mon Sep 17 00:00:00 2001 From: BLC Core Studio <207100624+BLCCoreStudio@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:42:23 +0300 Subject: [PATCH 8/8] ci: remove one-shot v1 version gate workflow --- .../workflows/one-shot-v1-version-gates.yml | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 .github/workflows/one-shot-v1-version-gates.yml diff --git a/.github/workflows/one-shot-v1-version-gates.yml b/.github/workflows/one-shot-v1-version-gates.yml deleted file mode 100644 index d2b9105..0000000 --- a/.github/workflows/one-shot-v1-version-gates.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: One-shot v1 version gate refresh - -on: - push: - branches: - - release/v1-readiness - paths: - - .github/workflows/one-shot-v1-version-gates.yml - -permissions: - contents: write - -jobs: - refresh-version-gates: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Checkout release branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: release/v1-readiness - fetch-depth: 0 - - - name: Align explicit v1 version assertions - run: | - python - <<'PY' - from pathlib import Path - - old = '__version__ == "0.13.0"' - new = '__version__ == "1.0.0"' - changed = [] - for path in Path('.github/workflows').glob('*.yml'): - if path.name == 'one-shot-v1-version-gates.yml': - continue - text = path.read_text(encoding='utf-8') - updated = text.replace(old, new) - if updated != text: - path.write_text(updated, encoding='utf-8') - changed.append(str(path)) - if not changed: - raise SystemExit('no pre-v1 version assertions found') - print('\n'.join(changed)) - PY - - - name: Verify no stale pre-v1 assertion remains - run: | - ! grep -R --line-number '__version__ == "0.13.0"' .github/workflows --include='*.yml' - - - name: Commit refreshed gates and remove this one-shot workflow - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add .github/workflows - git rm .github/workflows/one-shot-v1-version-gates.yml - git commit -m "ci: align workflow version assertions with v1" - git push origin HEAD:release/v1-readiness